From a1e3828c6e506825022681ecc3b5e4749db380ba Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:30:50 -0400 Subject: [PATCH 01/10] Update ruff and black config files --- .github/workflows/flake8.yml | 21 ----- .github/workflows/python-linting.yml | 11 +++ .pre-commit-config.yaml | 12 +++ pyproject.toml | 18 ++++ ruff.toml | 118 +++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 21 deletions(-) delete mode 100644 .github/workflows/flake8.yml create mode 100644 .github/workflows/python-linting.yml create mode 100644 .pre-commit-config.yaml create mode 100644 ruff.toml diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index 1ce62dc1..00000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Python flake8 check - -on: [push] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Lint with flake8 - run: | - pip install flake8 - # Note: only check files in Ska.engarchive package. Many other files in - # the repo are not maintained as PEP8 compliant. - flake8 agasc setup.py --count --ignore=E402,W503,F541 --max-line-length=100 --show-source --statistics diff --git a/.github/workflows/python-linting.yml b/.github/workflows/python-linting.yml new file mode 100644 index 00000000..2c5fccad --- /dev/null +++ b/.github/workflows/python-linting.yml @@ -0,0 +1,11 @@ +name: Check Python formatting using Black and Ruff + +on: [push] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable + - uses: chartboost/ruff-action@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..eeaa2f87 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: +- repo: https://github.com/psf/black + rev: 23.7.0 + hooks: + - id: black + language_version: "python3.10" + +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.292 + hooks: + - id: ruff + language_version: "python3.10" diff --git a/pyproject.toml b/pyproject.toml index baa1ce0f..c2d93d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,21 @@ ignore = [ "**/*.ipynb", ".git" ] + +[tool.black] +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.mypy_cache + | \.tox + | \.venv + | \.vscode + | \.eggs + | _build + | buck-out + | build + | dist + | docs +)/ +''' diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..7066aaad --- /dev/null +++ b/ruff.toml @@ -0,0 +1,118 @@ +# Copy ruff settings from pandas +[tool.ruff] +line-length = 100 +target-version = "py310" +# fix = true +unfixable = [] + +select = [ + # isort + "I", + # pyflakes + "F", + # pycodestyle + "E", "W", + # flake8-2020 + "YTT", + # flake8-bugbear + "B", + # flake8-quotes + "Q", + # flake8-debugger + "T10", + # flake8-gettext + "INT", + # pylint + "PLC", "PLE", "PLR", "PLW", + # misc lints + "PIE", + # flake8-pyi + "PYI", + # tidy imports + "TID", + # implicit string concatenation + "ISC", + # type-checking imports + "TCH", + # comprehensions + "C4", + # pygrep-hooks + "PGH" +] + +ignore = [ + # space before : (needed for how black formats slicing) + # "E203", # not yet implemented + # module level import not at top of file + "E402", + # do not assign a lambda expression, use a def + "E731", + # line break before binary operator + # "W503", # not yet implemented + # line break after binary operator + # "W504", # not yet implemented + # controversial + "B006", + # controversial?: Loop control variable not used within loop body + # "B007", + # controversial + "B008", + # setattr is used to side-step mypy + "B009", + # getattr is used to side-step mypy + "B010", + # tests use assert False + "B011", + # tests use comparisons but not their returned value + "B015", + # false positives + "B019", + # Loop control variable overrides iterable it iterates + "B020", + # Function definition does not bind loop variable + "B023", + # No explicit `stacklevel` keyword argument found + "B028", + # Functions defined inside a loop must not use variables redefined in the loop + # "B301", # not yet implemented + # Only works with python >=3.10 + "B905", + # Too many arguments to function call + "PLR0913", + # Too many returns + "PLR0911", + # Too many branches + "PLR0912", + # Too many statements + "PLR0915", + # Redefined loop name + "PLW2901", + # Global statements are discouraged + "PLW0603", + # Docstrings should not be included in stubs + "PYI021", + # No builtin `eval()` allowed + "PGH001", + # compare-to-empty-string + "PLC1901", + # Use typing_extensions.TypeAlias for type aliases + # "PYI026", # not yet implemented + # Use "collections.abc.*" instead of "typing.*" (PEP 585 syntax) + # "PYI027", # not yet implemented + # while int | float can be shortened to float, the former is more explicit + # "PYI041", # not yet implemented + + # Additional checks that don't pass yet + # Useless statement + "B018", + # Within an except clause, raise exceptions with ... + "B904", + # Magic number + "PLR2004", + # Consider `elif` instead of `else` then `if` to remove indentation level + "PLR5501", +] + +exclude = [ + "docs/", +] From 957cdebabc1c83fac69f866a318af1d2afba2366 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:34:08 -0400 Subject: [PATCH 02/10] Update black --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eeaa2f87..0ff7e639 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black language_version: "python3.10" From 73be6aa17b58004b74f7da3e56872a3fbab81279 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:35:03 -0400 Subject: [PATCH 03/10] Fix ruff config --- ruff.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 7066aaad..49b41cc9 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,5 +1,4 @@ # Copy ruff settings from pandas -[tool.ruff] line-length = 100 target-version = "py310" # fix = true From 5be3d969c88e8902b632c995389362ca3d2ea709 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:38:53 -0400 Subject: [PATCH 04/10] black --skip-string-normalization --preview . --- agasc/__init__.py | 1 + agasc/agasc.py | 122 +- agasc/paths.py | 1 + agasc/scripts/mag_estimate_report.py | 104 +- agasc/scripts/supplement_diff.py | 40 +- agasc/scripts/supplement_tasks.py | 44 +- agasc/scripts/update_mag_supplement.py | 159 ++- agasc/scripts/update_supplement.py | 128 +- agasc/supplement/magnitudes/mag_estimate.py | 1133 ++++++++++------- .../magnitudes/mag_estimate_report.py | 656 ++++++---- .../magnitudes/star_obs_catalogs.py | 8 +- .../magnitudes/update_mag_supplement.py | 306 +++-- agasc/supplement/utils.py | 148 ++- agasc/tests/test_agasc_1.py | 27 +- agasc/tests/test_agasc_2.py | 79 +- agasc/tests/test_obs_status.py | 1094 +++++++++++----- create_derived_agasc_h5.py | 4 +- create_near_neighbor_ids.py | 4 +- dev/profile_memory_psrecord.py | 2 + scripts/make_fits.py | 8 +- scripts/make_links.py | 1 - setup.py | 56 +- 22 files changed, 2659 insertions(+), 1466 deletions(-) diff --git a/agasc/__init__.py b/agasc/__init__.py index a0fa4a4f..93221cef 100644 --- a/agasc/__init__.py +++ b/agasc/__init__.py @@ -13,4 +13,5 @@ def test(*args, **kwargs): Run py.test unit tests. ''' import testr + return testr.test(*args, **kwargs) diff --git a/agasc/agasc.py b/agasc/agasc.py index 73a10ee2..85ede00a 100644 --- a/agasc/agasc.py +++ b/agasc/agasc.py @@ -17,10 +17,18 @@ from .paths import default_agasc_dir from .supplement.utils import get_supplement_table -__all__ = ['sphere_dist', 'get_agasc_cone', 'get_star', 'get_stars', 'read_h5_table', - 'get_agasc_filename', - 'MAG_CATID_SUPPLEMENT', 'BAD_CLASS_SUPPLEMENT', - 'set_supplement_enabled', 'SUPPLEMENT_ENABLED_ENV'] +__all__ = [ + 'sphere_dist', + 'get_agasc_cone', + 'get_star', + 'get_stars', + 'read_h5_table', + 'get_agasc_filename', + 'MAG_CATID_SUPPLEMENT', + 'BAD_CLASS_SUPPLEMENT', + 'set_supplement_enabled', + 'SUPPLEMENT_ENABLED_ENV', +] SUPPLEMENT_ENABLED_ENV = 'AGASC_SUPPLEMENT_ENABLED' SUPPLEMENT_ENABLED_DEFAULT = 'True' @@ -155,11 +163,11 @@ def get_ra_decs(agasc_file): def read_h5_table( - h5_file: str | Path | tables.file.File, - row0: Optional[int] = None, - row1: Optional[int] = None, - path="data", - cache=False, + h5_file: str | Path | tables.file.File, + row0: Optional[int] = None, + row1: Optional[int] = None, + path="data", + cache=False, ) -> np.ndarray: """ Read HDF5 table from group ``path`` in ``h5_file``. @@ -209,10 +217,10 @@ def _read_h5_table_cached( def _read_h5_table( - h5_file: str | Path | tables.file.File, - path: str, - row0: None | int, - row1: None | int, + h5_file: str | Path | tables.file.File, + path: str, + row0: None | int, + row1: None | int, ) -> np.ndarray: if isinstance(h5_file, tables.file.File): out = _read_h5_table_from_open_h5_file(h5_file, path, row0, row1) @@ -320,9 +328,9 @@ def get_agasc_filename( continue version_str = match.group(1) rc_str = match.group(2) or "" - if ( - version is not None - and version not in (version_str, version_str + rc_str) + if version is not None and version not in ( + version_str, + version_str + rc_str, ): continue matches.append((Version(version_str.replace("p", ".") + rc_str), path)) @@ -359,8 +367,10 @@ def sphere_dist(ra1, dec1, ra2, dec2): dec1 = np.radians(dec1).astype(np.float64) dec2 = np.radians(dec2).astype(np.float64) - numerator = numexpr.evaluate('sin((dec2 - dec1) / 2) ** 2 + ' # noqa - 'cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2') + numerator = numexpr.evaluate( + 'sin((dec2 - dec1) / 2) ** 2 + ' # noqa + 'cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2' + ) dists = numexpr.evaluate('2 * arctan2(numerator ** 0.5, (1 - numerator) ** 0.5)') return np.degrees(dists) @@ -418,23 +428,39 @@ def add_pmcorr_columns(stars, date): # ndarray float64 for consistent results between scalar and array cases. dyear = dates.frac_year - stars['EPOCH'].view(np.ndarray).astype(np.float64) - pm_to_degrees = dyear / (3600. * 1000.) - dec_pmcorr = np.where(stars['PM_DEC'] != -9999, - stars['DEC'] + stars['PM_DEC'] * pm_to_degrees, - stars['DEC']) + pm_to_degrees = dyear / (3600.0 * 1000.0) + dec_pmcorr = np.where( + stars['PM_DEC'] != -9999, + stars['DEC'] + stars['PM_DEC'] * pm_to_degrees, + stars['DEC'], + ) ra_scale = np.cos(np.radians(stars['DEC'])) - ra_pmcorr = np.where(stars['PM_RA'] != -9999, - stars['RA'] + stars['PM_RA'] * pm_to_degrees / ra_scale, - stars['RA']) + ra_pmcorr = np.where( + stars['PM_RA'] != -9999, + stars['RA'] + stars['PM_RA'] * pm_to_degrees / ra_scale, + stars['RA'], + ) # Add the proper-motion corrected columns to table using astropy.table.Table - stars.add_columns([Column(data=ra_pmcorr, name='RA_PMCORR'), - Column(data=dec_pmcorr, name='DEC_PMCORR')]) + stars.add_columns( + [ + Column(data=ra_pmcorr, name='RA_PMCORR'), + Column(data=dec_pmcorr, name='DEC_PMCORR'), + ] + ) -def get_agasc_cone(ra, dec, radius=1.5, date=None, agasc_file=None, - pm_filter=True, fix_color1=True, use_supplement=None, - cache=False): +def get_agasc_cone( + ra, + dec, + radius=1.5, + date=None, + agasc_file=None, + pm_filter=True, + fix_color1=True, + use_supplement=None, + cache=False, +): """ Get AGASC catalog entries within ``radius`` degrees of ``ra``, ``dec``. @@ -492,11 +518,11 @@ def get_agasc_cone(ra, dec, radius=1.5, date=None, agasc_file=None, def get_stars_from_dec_sorted_h5( - ra: float, - dec: float, - radius: float, - agasc_file: str | Path, - cache: bool = False, + ra: float, + dec: float, + radius: float, + agasc_file: str | Path, + cache: bool = False, ) -> Table: """ Returns a table of stars within a given radius of a given RA and Dec. @@ -577,7 +603,8 @@ def get_star(id, agasc_file=None, date=None, fix_color1=True, use_supplement=Non if len(id_rows) > 1: raise InconsistentCatalogError( - "More than one entry found for {} in AGASC".format(id)) + "More than one entry found for {} in AGASC".format(id) + ) if id_rows is None or len(id_rows) == 0: raise IdNotFound() @@ -601,7 +628,8 @@ def _get_rows_read_where(ids_1d, dates_1d, agasc_file): if len(id_rows) > 1: raise InconsistentCatalogError( - f'More than one entry found for {id} in AGASC') + f'More than one entry found for {id} in AGASC' + ) if id_rows is None or len(id_rows) == 0: raise IdNotFound(f'No entry found for {id} in AGASC') @@ -625,8 +653,14 @@ def _get_rows_read_entire(ids_1d, dates_1d, agasc_file): return rows -def get_stars(ids, agasc_file=None, dates=None, method_threshold=5000, fix_color1=True, - use_supplement=None): +def get_stars( + ids, + agasc_file=None, + dates=None, + method_threshold=5000, + fix_color1=True, + use_supplement=None, +): """ Get AGASC catalog entries for star ``ids`` at ``dates``. @@ -743,10 +777,14 @@ def update_from_supplement(stars, use_supplement=None): Use the supplement (default=None, see above) """ if use_supplement is None: - supplement_enabled_env = os.environ.get(SUPPLEMENT_ENABLED_ENV, SUPPLEMENT_ENABLED_DEFAULT) + supplement_enabled_env = os.environ.get( + SUPPLEMENT_ENABLED_ENV, SUPPLEMENT_ENABLED_DEFAULT + ) if supplement_enabled_env not in ('True', 'False'): - raise ValueError(f'{SUPPLEMENT_ENABLED_ENV} env var must be either "True" or "False" ' - f'got {supplement_enabled_env}') + raise ValueError( + f'{SUPPLEMENT_ENABLED_ENV} env var must be either "True" or "False" ' + f'got {supplement_enabled_env}' + ) supplement_enabled = supplement_enabled_env == 'True' else: supplement_enabled = use_supplement diff --git a/agasc/paths.py b/agasc/paths.py index c44d4027..1e377201 100644 --- a/agasc/paths.py +++ b/agasc/paths.py @@ -29,4 +29,5 @@ def default_agasc_file(): :returns: str """ from agasc import get_agasc_filename + return get_agasc_filename() diff --git a/agasc/scripts/mag_estimate_report.py b/agasc/scripts/mag_estimate_report.py index 167bd14b..e59d340e 100755 --- a/agasc/scripts/mag_estimate_report.py +++ b/agasc/scripts/mag_estimate_report.py @@ -16,34 +16,66 @@ def get_parser(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--start', help='Time to start processing new observations.' - ' CxoTime-compatible time stamp.' - ' Default: now - 90 days') - parser.add_argument('--stop', help='Time to stop processing new observations.' - ' CxoTime-compatible time stamp.' - ' Default: now') - parser.add_argument('--input-dir', default='$SKA/data/agasc', - help='Directory containing mag-stats files. Default: $SKA/data/agasc') - parser.add_argument('--output-dir', default='supplement_reports/suspect', - help='Output directory.' - ' Default: supplement_reports/suspect') - parser.add_argument('--obs-stats', default='mag_stats_obsid.fits', - help='FITS file with mag-stats for all observations.' - ' Default: mag_stats_obsid.fits') - parser.add_argument('--agasc-stats', default='mag_stats_agasc.fits', - help='FITS file with mag-stats for all observed AGASC stars.' - ' Default: mag_stats_agasc.fits') - parser.add_argument('--weekly-report', - help="Add links to navigate weekly reports.", - action='store_true', default=False) - parser.add_argument('--all-stars', - help="Include all stars in the report, not just suspect.", - action='store_true', default=False) + parser.add_argument( + '--start', + help=( + 'Time to start processing new observations.' + ' CxoTime-compatible time stamp.' + ' Default: now - 90 days' + ), + ) + parser.add_argument( + '--stop', + help=( + 'Time to stop processing new observations.' + ' CxoTime-compatible time stamp.' + ' Default: now' + ), + ) + parser.add_argument( + '--input-dir', + default='$SKA/data/agasc', + help='Directory containing mag-stats files. Default: $SKA/data/agasc', + ) + parser.add_argument( + '--output-dir', + default='supplement_reports/suspect', + help='Output directory. Default: supplement_reports/suspect', + ) + parser.add_argument( + '--obs-stats', + default='mag_stats_obsid.fits', + help=( + 'FITS file with mag-stats for all observations.' + ' Default: mag_stats_obsid.fits' + ), + ) + parser.add_argument( + '--agasc-stats', + default='mag_stats_agasc.fits', + help=( + 'FITS file with mag-stats for all observed AGASC stars.' + ' Default: mag_stats_agasc.fits' + ), + ) + parser.add_argument( + '--weekly-report', + help="Add links to navigate weekly reports.", + action='store_true', + default=False, + ) + parser.add_argument( + '--all-stars', + help="Include all stars in the report, not just suspect.", + action='store_true', + default=False, + ) return parser def main(): import kadi.commands + kadi.commands.conf.commands_version = '1' args = get_parser().parse_args() @@ -64,16 +96,12 @@ def main(): else: args.start = CxoTime(args.start) - t = (obs_stats['mp_starcat_time']) + t = obs_stats['mp_starcat_time'] ok = (t < args.stop) & (t > args.start) if not args.all_stars: ok &= ~obs_stats['obs_ok'] stars = np.unique(obs_stats[ok]['agasc_id']) - sections = [{ - 'id': 'stars', - 'title': 'Stars', - 'stars': stars - }] + sections = [{'id': 'stars', 'title': 'Stars', 'stars': stars}] agasc_stats = agasc_stats[np.in1d(agasc_stats['agasc_id'], stars)] @@ -84,18 +112,22 @@ def main(): nav_links = { 'previous': f'../{(t - week).date[:8]}', 'up': '..', - 'next': f'../{(t + week).date[:8]}' + 'next': f'../{(t + week).date[:8]}', } else: directory = args.output_dir nav_links = None - msr = mag_estimate_report.MagEstimateReport(agasc_stats, - obs_stats, - directory=directory) - msr.multi_star_html(sections=sections, tstart=args.start, tstop=args.stop, - filename='index.html', - nav_links=nav_links) + msr = mag_estimate_report.MagEstimateReport( + agasc_stats, obs_stats, directory=directory + ) + msr.multi_star_html( + sections=sections, + tstart=args.start, + tstop=args.stop, + filename='index.html', + nav_links=nav_links, + ) if __name__ == '__main__': diff --git a/agasc/scripts/supplement_diff.py b/agasc/scripts/supplement_diff.py index 5180023b..b3fc8db0 100755 --- a/agasc/scripts/supplement_diff.py +++ b/agasc/scripts/supplement_diff.py @@ -23,10 +23,7 @@ def read_file(filename, exclude=[]): formats = { 'agasc_versions': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']}, 'last_updated': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']}, - 'obs': { - 'comments': '{:>80s}', - 'agasc_id': '{:10d}' - }, + 'obs': {'comments': '{:>80s}', 'agasc_id': '{:10d}'}, } node_names = ['agasc_versions', 'last_updated', 'obs', 'bad'] @@ -46,7 +43,7 @@ def read_file(filename, exclude=[]): s.seek(0) lines = s.readlines() dashes = "-" * (len(lines[0]) - 1) + "\n" - all_lines += ([dashes, f'| {node.name}\n', dashes] + lines + [dashes]) + all_lines += [dashes, f'| {node.name}\n', dashes] + lines + [dashes] return all_lines @@ -63,23 +60,26 @@ def diff_to_html(diff): date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') lexer = pygments.lexers.DiffLexer() formatter = pygments.formatters.HtmlFormatter( - full=True, linenos='table', - title=f'AGASC supplement diff - {date}') + full=True, linenos='table', title=f'AGASC supplement diff - {date}' + ) return pygments.highlight(diff, lexer, formatter) def get_parser(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--from', dest='fromfile', type=Path, help='The original supplement file.') - parser.add_argument('--to', dest='tofile', type=Path, help='The modified supplement file.') parser.add_argument( - '-o', - help='Output HTML file', - type=Path, - default='agasc_supplement_diff.html' + '--from', dest='fromfile', type=Path, help='The original supplement file.' + ) + parser.add_argument( + '--to', dest='tofile', type=Path, help='The modified supplement file.' + ) + parser.add_argument( + '-o', help='Output HTML file', type=Path, default='agasc_supplement_diff.html' ) parser.add_argument( - '--exclude-mags', default=False, action='store_true', + '--exclude-mags', + default=False, + action='store_true', help='Exclude changes in the "mags" table. There can be many of these.', ) return parser @@ -88,9 +88,13 @@ def get_parser(): def main(): args = get_parser().parse_args() if args.fromfile is None and 'SKA' in os.environ: - args.fromfile = Path(os.environ['SKA']) / 'data' / 'agasc' / 'agasc_supplement.h5' + args.fromfile = ( + Path(os.environ['SKA']) / 'data' / 'agasc' / 'agasc_supplement.h5' + ) if args.tofile is None and 'SKA' in os.environ: - args.tofile = Path(os.environ['SKA']) / 'data' / 'agasc' / 'rc' / 'agasc_supplement.h5' + args.tofile = ( + Path(os.environ['SKA']) / 'data' / 'agasc' / 'rc' / 'agasc_supplement.h5' + ) assert args.tofile, 'Option "--to" was not given and SKA is not defined' assert args.fromfile, 'Option "--from" was not given and SKA is not defined' assert args.fromfile.exists(), f'File {args.fromfile} does not exist' @@ -100,7 +104,9 @@ def main(): args.o.parent.mkdir() with open(args.o, 'w') as fh: - diff = diff_to_html(diff_files(args.fromfile, args.tofile, exclude_mags=args.exclude_mags)) + diff = diff_to_html( + diff_files(args.fromfile, args.tofile, exclude_mags=args.exclude_mags) + ) fh.write(diff) diff --git a/agasc/scripts/supplement_tasks.py b/agasc/scripts/supplement_tasks.py index ab755aa9..17694c73 100644 --- a/agasc/scripts/supplement_tasks.py +++ b/agasc/scripts/supplement_tasks.py @@ -25,15 +25,14 @@ def email_promotion_report( - filenames, - destdir, - to, - sender=f"{getpass.getuser()}@{platform.uname()[1]}" + filenames, destdir, to, sender=f"{getpass.getuser()}@{platform.uname()[1]}" ): date = CxoTime().date[:14] filenames = "- " + "\n- ".join([str(f) for f in filenames]) - msg = MIMEText(f"The following files were promoted to {destdir} on {date}:\n{filenames}") + msg = MIMEText( + f"The following files were promoted to {destdir} on {date}:\n{filenames}" + ) msg["From"] = sender msg["To"] = to msg["Subject"] = "AGASC RC supplement promoted" @@ -49,17 +48,11 @@ def update_rc(): if filenames: for file in filenames: file.rename(AGASC_DATA / file.name) - email_promotion_report( - filenames, - destdir=AGASC_DATA, - to='aca@cfa.harvard.edu' - ) + email_promotion_report(filenames, destdir=AGASC_DATA, to='aca@cfa.harvard.edu') - subprocess.run([ - 'task_schedule3.pl', - '-config', - 'agasc/task_schedule_update_supplement_rc.cfg' - ]) + subprocess.run( + ['task_schedule3.pl', '-config', 'agasc/task_schedule_update_supplement_rc.cfg'] + ) def disposition(): @@ -68,11 +61,13 @@ def disposition(): This actually schedules a task to run. """ - subprocess.run([ - 'task_schedule3.pl', - '-config', - 'agasc/task_schedule_supplement_dispositions.cfg' - ]) + subprocess.run( + [ + 'task_schedule3.pl', + '-config', + 'agasc/task_schedule_supplement_dispositions.cfg', + ] + ) def stage_promotion(): @@ -85,7 +80,11 @@ def stage_promotion(): promote_dir = AGASC_DATA / 'rc' / 'promote' rc_dir = AGASC_DATA / 'rc' promote_dir.mkdir(exist_ok=True) - for filename in ['agasc_supplement.h5', 'mag_stats_agasc.fits', 'mag_stats_obsid.fits']: + for filename in [ + 'agasc_supplement.h5', + 'mag_stats_agasc.fits', + 'mag_stats_obsid.fits', + ]: shutil.copy(rc_dir / filename, promote_dir / filename) @@ -98,8 +97,7 @@ def stage_promotion(): def get_parser(): parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument('task', choices=TASKS) return parser diff --git a/agasc/scripts/update_mag_supplement.py b/agasc/scripts/update_mag_supplement.py index 8e59bf89..5ca46ecd 100755 --- a/agasc/scripts/update_mag_supplement.py +++ b/agasc/scripts/update_mag_supplement.py @@ -20,59 +20,100 @@ def get_parser(): parser = argparse.ArgumentParser( - description=__doc__, - parents=[update_supplement.get_obs_status_parser()] + description=__doc__, parents=[update_supplement.get_obs_status_parser()] + ) + parser.add_argument( + '--start', + help=( + 'Include only stars observed after this time.' + ' CxoTime-compatible time stamp.' + ' Default: now - 30 days.' + ), + ) + parser.add_argument( + '--stop', + help=( + 'Include only stars observed before this time.' + ' CxoTime-compatible time stamp.' + ' Default: now.' + ), + ) + parser.add_argument( + '--whole-history', + help='Include all star observations and ignore --start/stop.', + action='store_true', + default=False, + ) + parser.add_argument( + '--agasc-id-file', + help=( + 'Include only observations of stars whose AGASC IDs are specified ' + 'in this file, one per line.' + ), + ) + parser.add_argument( + '--output-dir', + help=( + 'Directory where agasc_supplement.h5 is located.' + 'Other output is placed here as well. Default: .' + ), + default='.', + ) + parser.add_argument( + '--include-bad', + help='Do not exclude "bad" stars from magnitude estimates. Default: False', + action='store_true', + default=False, ) - parser.add_argument('--start', - help='Include only stars observed after this time.' - ' CxoTime-compatible time stamp.' - ' Default: now - 30 days.') - parser.add_argument('--stop', - help='Include only stars observed before this time.' - ' CxoTime-compatible time stamp.' - ' Default: now.') - parser.add_argument('--whole-history', - help='Include all star observations and ignore --start/stop.', - action='store_true', default=False) - parser.add_argument('--agasc-id-file', - help='Include only observations of stars whose AGASC IDs are specified ' - 'in this file, one per line.') - parser.add_argument('--output-dir', - help='Directory where agasc_supplement.h5 is located.' - 'Other output is placed here as well. Default: .', - default='.') - parser.add_argument('--include-bad', - help='Do not exclude "bad" stars from magnitude estimates. Default: False', - action='store_true', default=False) report = parser.add_argument_group('Reporting') - report.add_argument('--report', - help='Generate HTML report for the period covered. Default: False', - action='store_true', default=False) - report.add_argument('--reports-dir', - help='Directory where to place reports.' - ' Default: /supplement_reports/weekly.') + report.add_argument( + '--report', + help='Generate HTML report for the period covered. Default: False', + action='store_true', + default=False, + ) + report.add_argument( + '--reports-dir', + help=( + 'Directory where to place reports.' + ' Default: /supplement_reports/weekly.' + ), + ) other = parser.add_argument_group('Other') - other.add_argument('--multi-process', - help="Use multi-processing to accelerate run.", - action='store_true', default=False) - other.add_argument('--log-level', - default='info', - choices=['debug', 'info', 'warning', 'error']) - other.add_argument('--no-progress', dest='no_progress', - help='Do not show a progress bar', - action='store_true') # this has no default, it will be None. - other.add_argument('--args-file', - help='YAML file with arguments to ' - 'agasc.supplement.magnitudes.update_mag_supplement.do') - other.add_argument("--dry-run", - action="store_true", - help="Dry run (no actual file or database updates)") + other.add_argument( + '--multi-process', + help="Use multi-processing to accelerate run.", + action='store_true', + default=False, + ) + other.add_argument( + '--log-level', default='info', choices=['debug', 'info', 'warning', 'error'] + ) + other.add_argument( + '--no-progress', + dest='no_progress', + help='Do not show a progress bar', + action='store_true', + ) # this has no default, it will be None. + other.add_argument( + '--args-file', + help=( + 'YAML file with arguments to ' + 'agasc.supplement.magnitudes.update_mag_supplement.do' + ), + ) + other.add_argument( + "--dry-run", + action="store_true", + help="Dry run (no actual file or database updates)", + ) return parser def main(): import kadi.commands + kadi.commands.conf.commands_version = '1' logger = logging.getLogger('agasc.supplement') @@ -100,7 +141,9 @@ def main(): if args.whole_history: if args.start or args.stop: - logger.error('--whole-history argument is incompatible with --start/--stop arguments') + logger.error( + '--whole-history argument is incompatible with --start/--stop arguments' + ) the_parser.exit(1) args.start = None args.stop = None @@ -108,13 +151,16 @@ def main(): pyyaks.logger.get_logger( name='agasc.supplement', level=args.log_level.upper(), - format="%(asctime)s %(message)s" + format="%(asctime)s %(message)s", ) - if (((args.obsid or args.mp_starcat_time) and not args.status) - or (not (args.obsid or args.mp_starcat_time) and args.status)): + if ((args.obsid or args.mp_starcat_time) and not args.status) or ( + not (args.obsid or args.mp_starcat_time) and args.status + ): logger.error( - 'To override OBS status, both --obs/mp-starcat-time and --status options are needed.') + 'To override OBS status, both --obs/mp-starcat-time and --status options' + ' are needed.' + ) the_parser.exit(1) star_obs_catalogs.load(args.stop) @@ -136,7 +182,11 @@ def main(): args.stop = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).max().date else: args.stop = CxoTime(args.stop).date if args.stop else CxoTime.now().date - args.start = CxoTime(args.start).date if args.start else (CxoTime.now() - 30 * u.day).date + args.start = ( + CxoTime(args.start).date + if args.start + else (CxoTime.now() - 30 * u.day).date + ) report_date = None if args.report: @@ -151,8 +201,9 @@ def main(): args.output_dir.mkdir(parents=True) with open(args_log_file, 'w') as fh: # there must be a better way to do this... - yaml_args = {k: str(v) if issubclass(type(v), Path) else v - for k, v in vars(args).items()} + yaml_args = { + k: str(v) if issubclass(type(v), Path) else v for k, v in vars(args).items() + } yaml.dump(yaml_args, fh) update_mag_supplement.do( @@ -169,7 +220,9 @@ def main(): no_progress=args.no_progress, ) if args.report and (args.reports_dir / f'{report_date.date[:8]}').exists(): - args_log_file.replace(args.reports_dir / f'{report_date.date[:8]}' / args_log_file.name) + args_log_file.replace( + args.reports_dir / f'{report_date.date[:8]}' / args_log_file.name + ) if __name__ == '__main__': diff --git a/agasc/scripts/update_supplement.py b/agasc/scripts/update_supplement.py index 856da197..aa448bf9 100755 --- a/agasc/scripts/update_supplement.py +++ b/agasc/scripts/update_supplement.py @@ -55,7 +55,9 @@ def _sanitize_args(status): else: obs = cat.STARS_OBS[cat.STARS_OBS['agasc_id'] == star['agasc_id']] if len(obs) == 0: - raise Exception(f"Can not guess last_obs_time for agasc_id={star['agasc_id']}") + raise Exception( + f"Can not guess last_obs_time for agasc_id={star['agasc_id']}" + ) star['last_obs_time'] = CxoTime(obs['mp_starcat_time']).max().cxcsec for value in status['obs']: @@ -63,7 +65,9 @@ def _sanitize_args(status): raise RuntimeError('Observation catalog is not initialized') if 'mp_starcat_time' in value: key = 'mp_starcat_time' - rows = cat.STARS_OBS[cat.STARS_OBS['mp_starcat_time'] == value['mp_starcat_time']] + rows = cat.STARS_OBS[ + cat.STARS_OBS['mp_starcat_time'] == value['mp_starcat_time'] + ] elif 'obsid' in value: key = 'obsid' rows = cat.STARS_OBS[cat.STARS_OBS['obsid'] == value['obsid']] @@ -77,7 +81,7 @@ def _sanitize_args(status): if 'agasc_id' not in value: value['agasc_id'] = list(sorted(rows['agasc_id'])) else: - value['agasc_id'] = (list(np.atleast_1d(value['agasc_id']))) + value['agasc_id'] = list(np.atleast_1d(value['agasc_id'])) if 'comments' not in value: value['comments'] = '' if 'mp_starcat_time' not in value: @@ -104,10 +108,17 @@ def _sanitize_args(status): return status -def parse_args(filename=None, bad_star_id=None, bad_star_source=None, - obsid=None, status=None, comments='', agasc_id=None, mp_starcat_time=None, - **_ - ): +def parse_args( + filename=None, + bad_star_id=None, + bad_star_source=None, + obsid=None, + status=None, + comments='', + agasc_id=None, + mp_starcat_time=None, + **_, +): """ Combine obs/bad-star status from file and from arguments. @@ -145,17 +156,23 @@ def parse_args(filename=None, bad_star_id=None, bad_star_source=None, 'agasc_id': agasc_id, 'obsid': obsid, 'status': status, - 'comments': comments + 'comments': comments, } optional = ['obsid', 'mp_starcat_time', 'agasc_id', 'comments'] obs_status_override.append( - {key: row[key] for key in row if key not in optional or row[key] is not None} + { + key: row[key] + for key in row + if key not in optional or row[key] is not None + } ) for bs in bad_star_id: bad_dict = dict(bad) if bs in bad_dict and bad_dict[bs] != bad_star_source: - raise RuntimeError('name collision: conflicting bad_star in file and in args') + raise RuntimeError( + 'name collision: conflicting bad_star in file and in args' + ) if (bs, bad_star_source) not in bad: bad.append((bs, bad_star_source)) @@ -173,23 +190,40 @@ def get_obs_status_parser(): parser = argparse.ArgumentParser(add_help=False) status = parser.add_argument_group( 'OBS/star status', - 'options to modify the "bads" and "obs" tables in AGASC supplement. ' - 'Modifications to supplement happen before all magnitude estimates are made.' + ( + 'options to modify the "bads" and "obs" tables in AGASC supplement.' + ' Modifications to supplement happen before all magnitude estimates are' + ' made.' + ), + ) + status.add_argument( + '--obs-status-file', + help=( + 'YAML file with star/observation status. ' + 'More info at https://sot.github.io/agasc/supplement.html' + ), + ) + status.add_argument( + '--mp-starcat-time', + help=( + 'Observation starcat time for status override. ' + 'Usually the mission planning catalog time' + ), ) - status.add_argument('--obs-status-file', - help='YAML file with star/observation status. ' - 'More info at https://sot.github.io/agasc/supplement.html') - status.add_argument('--mp-starcat-time', - help='Observation starcat time for status override. ' - 'Usually the mission planning catalog time') status.add_argument('--obsid', help='OBSID for status override.', type=int) status.add_argument('--agasc-id', help='AGASC ID for status override.', type=int) status.add_argument('--status', help='Status to override.') status.add_argument('--comments', help='Comments for status override.', default='') - status.add_argument('--bad-star-id', help='AGASC ID of bad star.', - default=[], action='append', type=int) - status.add_argument("--bad-star-source", type=int, - help="Source identifier indicating provenance.") + status.add_argument( + '--bad-star-id', + help='AGASC ID of bad star.', + default=[], + action='append', + type=int, + ) + status.add_argument( + "--bad-star-source", type=int, help="Source identifier indicating provenance." + ) return parser @@ -201,19 +235,22 @@ def get_parser(): :return: argparse.ArgumentParser """ parse = argparse.ArgumentParser( - description=__doc__, - parents=[get_obs_status_parser()] + description=__doc__, parents=[get_obs_status_parser()] + ) + parse.add_argument( + "--output-dir", + default='.', + type=Path, + help="Directory containing agasc_supplement.h5 (default='.')", + ) + parse.add_argument( + '--log-level', default='info', choices=['debug', 'info', 'warning', 'error'] + ) + parse.add_argument( + "--dry-run", + action="store_true", + help="Dry run (no actual file or database updates)", ) - parse.add_argument("--output-dir", - default='.', - type=Path, - help="Directory containing agasc_supplement.h5 (default='.')") - parse.add_argument('--log-level', - default='info', - choices=['debug', 'info', 'warning', 'error']) - parse.add_argument("--dry-run", - action="store_true", - help="Dry run (no actual file or database updates)") return parse @@ -225,34 +262,28 @@ def update(args): Returns a list of AGASC IDs whose records were updated in the supplement """ - status = parse_args( - filename=args.obs_status_file, - **vars(args) - ) + status = parse_args(filename=args.obs_status_file, **vars(args)) if status['mags']: update_mags_table( args.output_dir / 'agasc_supplement.h5', status['mags'], - dry_run=args.dry_run + dry_run=args.dry_run, ) if status['obs']: update_obs_table( - args.output_dir / 'agasc_supplement.h5', - status['obs'], - dry_run=args.dry_run + args.output_dir / 'agasc_supplement.h5', status['obs'], dry_run=args.dry_run ) if status['bad']: add_bad_star( - args.output_dir / 'agasc_supplement.h5', - status['bad'], - dry_run=args.dry_run + args.output_dir / 'agasc_supplement.h5', status['bad'], dry_run=args.dry_run ) - agasc_ids = sorted(set([o['agasc_id'] for o in status['obs']] - + [o[0] for o in status['bad']])) + agasc_ids = sorted( + set([o['agasc_id'] for o in status['obs']] + [o[0] for o in status['bad']]) + ) return agasc_ids @@ -261,6 +292,7 @@ def main(): The main function for the update_obs_status script. """ import kadi.commands + kadi.commands.conf.commands_version = '1' args = get_parser().parse_args() @@ -272,7 +304,7 @@ def main(): pyyaks.logger.get_logger( name='agasc.supplement', level=args.log_level.upper(), - format="%(asctime)s %(message)s" + format="%(asctime)s %(message)s", ) cat.load() diff --git a/agasc/supplement/magnitudes/mag_estimate.py b/agasc/supplement/magnitudes/mag_estimate.py index 3ea6887b..89bc2f82 100644 --- a/agasc/supplement/magnitudes/mag_estimate.py +++ b/agasc/supplement/magnitudes/mag_estimate.py @@ -32,14 +32,18 @@ MAX_MAG = 15 MASK = { - 'mouse_bit': np.array([[True, True, True, True, True, True, True, True], - [True, True, False, False, False, False, True, True], - [True, False, False, False, False, False, False, True], - [True, False, False, False, False, False, False, True], - [True, False, False, False, False, False, False, True], - [True, False, False, False, False, False, False, True], - [True, True, False, False, False, False, True, True], - [True, True, True, True, True, True, True, True]]) + 'mouse_bit': np.array( + [ + [True, True, True, True, True, True, True, True], + [True, True, False, False, False, False, True, True], + [True, False, False, False, False, False, False, True], + [True, False, False, False, False, False, False, True], + [True, False, False, False, False, False, False, True], + [True, False, False, False, False, False, False, True], + [True, True, False, False, False, False, True, True], + [True, True, True, True, True, True, True, True], + ] + ) } @@ -51,49 +55,49 @@ 3: 'Mismatch in telemetry between aca_l0 and cheta', 4: 'Time mismatch between cheta and level0', 5: 'Failed job', - 6: 'Suspect observation' + 6: 'Suspect observation', } EXCEPTION_CODES = collections.defaultdict(lambda: -1) EXCEPTION_CODES.update({msg: code for code, msg in EXCEPTION_MSG.items() if code > 0}) class MagStatsException(Exception): - def __init__(self, msg='', agasc_id=None, obsid=None, mp_starcat_time=None, - **kwargs): + def __init__( + self, msg='', agasc_id=None, obsid=None, mp_starcat_time=None, **kwargs + ): super().__init__(msg) self.error_code = EXCEPTION_CODES[msg] self.msg = msg self.agasc_id = agasc_id self.obsid = obsid[0] if type(obsid) is list and len(obsid) == 1 else obsid - self.mp_starcat_time = (mp_starcat_time[0] if type(mp_starcat_time) is list - and len(mp_starcat_time) == 1 else mp_starcat_time) + self.mp_starcat_time = ( + mp_starcat_time[0] + if type(mp_starcat_time) is list and len(mp_starcat_time) == 1 + else mp_starcat_time + ) for k in kwargs: setattr(self, k, kwargs[k]) exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is None: - self.exception = { - 'type': '', - 'value': '', - 'traceback': '' - } + self.exception = {'type': '', 'value': '', 'traceback': ''} else: stack_trace = [] for step in traceback.extract_tb(exc_traceback): - stack_trace.append( - f" in {step.filename}:{step.lineno}/{step.name}:" - ) + stack_trace.append(f" in {step.filename}:{step.lineno}/{step.name}:") stack_trace.append(f" {step.line}") self.exception = { 'type': exc_type.__name__, 'value': str(exc_value), - 'traceback': '\n'.join(stack_trace) + 'traceback': '\n'.join(stack_trace), } def __str__(self): - return f'MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, ' \ - f'obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})' + return ( + f'MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, ' + f'obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})' + ) def __iter__(self): yield 'error_code', self.error_code @@ -113,17 +117,22 @@ def _magnitude_correction(time, mag_aca): :param mag_aca: np.array :return: np.array """ - params = {"t_ref": "2011-01-01 12:00:00.000", - "p": [0.005899340720522751, - 0.12029019332761458, - -2.99386247406073e-10, - -6.9534637950633265, - 0.7916261423307238]} + params = { + "t_ref": "2011-01-01 12:00:00.000", + "p": [ + 0.005899340720522751, + 0.12029019332761458, + -2.99386247406073e-10, + -6.9534637950633265, + 0.7916261423307238, + ], + } q = params['p'] t_ref = DateTime(params['t_ref']) - dmag = (q[0] + (q[1] + q[2] * np.atleast_1d(time)) - * np.exp(q[3] + q[4] * np.atleast_1d(mag_aca))) + dmag = q[0] + (q[1] + q[2] * np.atleast_1d(time)) * np.exp( + q[3] + q[4] * np.atleast_1d(mag_aca) + ) dmag[np.atleast_1d(time) < t_ref.secs] = 0 return np.squeeze(dmag) @@ -141,8 +150,8 @@ def get_responsivity(time): Time in CXC seconds :return: """ - a, b, c = [3.19776750e-02, 5.35201479e+08, 8.49670756e+07] - return - a * (1 + scipy.special.erf((time - b) / c)) / 2 + a, b, c = [3.19776750e-02, 5.35201479e08, 8.49670756e07] + return -a * (1 + scipy.special.erf((time - b) / c)) / 2 def get_droop_systematic_shift(magnitude): @@ -226,25 +235,29 @@ def get_star_position(star, telem): aca_misalign = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) rad_to_arcsec = 206264.81 - q = np.array([telem['AOATTQT1'], - telem['AOATTQT2'], - telem['AOATTQT3'], - telem['AOATTQT4']]).transpose() + q = np.array( + [telem['AOATTQT1'], telem['AOATTQT2'], telem['AOATTQT3'], telem['AOATTQT4']] + ).transpose() norm = np.sum(q**2, axis=1, keepdims=True) # I am just normalizing q, just in case. n = np.squeeze(np.sqrt(norm)) - q[n != 0] /= np.sqrt(norm)[n != 0] # prevent warning when dividing by zero (it happens) + q[n != 0] /= np.sqrt(norm)[ + n != 0 + ] # prevent warning when dividing by zero (it happens) q_att = Quat(q=q) ts = q_att.transform star_pos_eci = Ska.quatutil.radec2eci(star['RA_PMCORR'], star['DEC_PMCORR']) - d_aca = np.dot(np.dot(aca_misalign, ts.transpose(0, 2, 1)), - star_pos_eci).transpose() + d_aca = np.dot( + np.dot(aca_misalign, ts.transpose(0, 2, 1)), star_pos_eci + ).transpose() yag = np.arctan2(d_aca[:, 1], d_aca[:, 0]) * rad_to_arcsec zag = np.arctan2(d_aca[:, 2], d_aca[:, 0]) * rad_to_arcsec - logger.debug(f' star position. AGASC_ID={star["AGASC_ID"]}, ' - f'{len(yag)} samples, ({yag[0]}, {zag[0]})...') + logger.debug( + f' star position. AGASC_ID={star["AGASC_ID"]}, ' + f'{len(yag)} samples, ({yag[0]}, {zag[0]})...' + ) return { 'yang_star': yag, 'zang_star': zag, @@ -252,32 +265,34 @@ def get_star_position(star, telem): # this is in case one has to return empty telemetry -_telem_dtype = [('times', 'float64'), - ('IMGSIZE', 'int32'), - ('IMGROW0', 'int16'), - ('IMGCOL0', 'int16'), - ('IMGRAW', 'float32'), - ('AOACASEQ', '= excluded_range[0]) & (times <= excluded_range[1])) + excluded |= (times >= excluded_range[0]) & (times <= excluded_range[1]) telem.update({k: telem[k][~excluded] for k in telem}) slot_data = slot_data[~excluded] if len(slot_data) == 0: # the intersection was null. - raise MagStatsException('Nothing left after removing excluded ranges', - agasc_id=obs["agasc_id"], - obsid=obs["obsid"], - mp_starcat_time=obs["mp_starcat_time"]) + raise MagStatsException( + 'Nothing left after removing excluded ranges', + agasc_id=obs["agasc_id"], + obsid=obs["obsid"], + mp_starcat_time=obs["mp_starcat_time"], + ) for name in ['AOACIIR', 'AOACISP', 'AOACYAN', 'AOACZAN', 'AOACMAG', 'AOACFCT']: telem[name] = telem[f'{name}{slot}'] del telem[f'{name}{slot}'] for name in ['AOACIIR', 'AOACISP']: telem[name] = np.char.rstrip(telem[name]) - mag_est_ok = \ - (telem['AOACASEQ'] == 'KALM') & (telem['AOACIIR'] == 'OK') & \ - (telem['AOPCADMD'] == 'NPNT') & (telem['AOACFCT'] == 'TRAK') + mag_est_ok = ( + (telem['AOACASEQ'] == 'KALM') + & (telem['AOACIIR'] == 'OK') + & (telem['AOPCADMD'] == 'NPNT') + & (telem['AOACFCT'] == 'TRAK') + ) - assert len(slot_data) == len(mag_est_ok), \ - f'len(slot_data) != len(ok) ({len(slot_data)} != {len(mag_est_ok)})' + assert len(slot_data) == len( + mag_est_ok + ), f'len(slot_data) != len(ok) ({len(slot_data)} != {len(mag_est_ok)})' # etc... logger.debug(' Adding magnitude estimates') @@ -397,7 +440,7 @@ def get_telemetry(obs): logger.debug(' Correcting for responsivity') responsivity = get_responsivity(start) telem['mags'] = telem['mags_img'] - responsivity - droop_shift - telem['mags'][~mag_est_ok] = 0. + telem['mags'][~mag_est_ok] = 0.0 telem['mag_est_ok'] = mag_est_ok telem['dy'] = np.ones(len(mag_est_ok)) * np.inf @@ -409,14 +452,16 @@ def get_telemetry(obs): if np.any(mag_est_ok & (rang < 10)): y25, y50, y75 = np.quantile(yang[mag_est_ok & (rang < 10)], [0.25, 0.5, 0.75]) z25, z50, z75 = np.quantile(zang[mag_est_ok & (rang < 10)], [0.25, 0.5, 0.75]) - centroid_outlier = ((yang > y75 + 3 * (y75 - y25)) - | (yang < y25 - 3 * (y75 - y25)) - | (zang > z75 + 3 * (z75 - z25)) - | (zang < z25 - 3 * (z75 - z25))) + centroid_outlier = ( + (yang > y75 + 3 * (y75 - y25)) + | (yang < y25 - 3 * (y75 - y25)) + | (zang > z75 + 3 * (z75 - z25)) + | (zang < z25 - 3 * (z75 - z25)) + ) telem['dy'] = yang - np.mean(yang[mag_est_ok & ~centroid_outlier]) telem['dz'] = zang - np.mean(zang[mag_est_ok & ~centroid_outlier]) - telem['dr'] = (telem['dy'] ** 2 + telem['dz'] ** 2) ** .5 + telem['dr'] = (telem['dy'] ** 2 + telem['dz'] ** 2) ** 0.5 return telem @@ -437,10 +482,13 @@ def get_telemetry_by_agasc_id(agasc_id, obsid=None, ignore_exceptions=False): star_obs_catalogs.load() if obsid is None: obs = star_obs_catalogs.STARS_OBS[ - (star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id)] + (star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id) + ] else: - obs = star_obs_catalogs.STARS_OBS[(star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id) - & (star_obs_catalogs.STARS_OBS['obsid'] == obsid)] + obs = star_obs_catalogs.STARS_OBS[ + (star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id) + & (star_obs_catalogs.STARS_OBS['obsid'] == obsid) + ] if len(obs) > 1: obs = obs.loc['mp_starcat_time', sorted(obs['mp_starcat_time'])] telem = [] @@ -489,15 +537,22 @@ def add_obs_info(telem, obs_stats): for s in obs_stats: obsid = s['obsid'] - o = (telem['obsid'] == obsid) + o = telem['obsid'] == obsid telem['obs_ok'][o] = np.ones(np.count_nonzero(o), dtype=bool) * s['obs_ok'] - if (np.any(telem['mag_est_ok'][o]) and s['f_mag_est_ok'] > 0 - and np.isfinite(s['q75']) and np.isfinite(s['q25'])): + if ( + np.any(telem['mag_est_ok'][o]) + and s['f_mag_est_ok'] > 0 + and np.isfinite(s['q75']) + and np.isfinite(s['q25']) + ): iqr = s['q75'] - s['q25'] telem['obs_outlier'][o] = ( - telem[o]['mag_est_ok'] & (iqr > 0) - & ((telem[o]['mags'] < s['q25'] - 1.5 * iqr) - | (telem[o]['mags'] > s['q75'] + 1.5 * iqr)) + telem[o]['mag_est_ok'] + & (iqr > 0) + & ( + (telem[o]['mags'] < s['q25'] - 1.5 * iqr) + | (telem[o]['mags'] > s['q75'] + 1.5 * iqr) + ) ) logger.debug(f' Adding observation info to telemetry {obsid=}') return telem @@ -507,7 +562,7 @@ def add_obs_info(telem, obs_stats): def staggered_aca_slice(array_in, array_out, row, col): for i in np.arange(len(row)): if row[i] + 8 < 1024 and col[i] + 8 < 1024: - array_out[i] = array_in[row[i]:row[i] + 8, col[i]:col[i] + 8] + array_out[i] = array_in[row[i] : row[i] + 8, col[i] : col[i] + 8] def get_mag_from_img(slot_data, t_start, ok=True): @@ -524,23 +579,26 @@ def get_mag_from_img(slot_data, t_start, ok=True): :return: """ logger.debug(' magnitude from images...') - dark_cal = get_dark_cal_image(t_start, 'nearest', - t_ccd_ref=np.mean(slot_data['TEMPCCD'] - 273.16), - aca_image=False) + dark_cal = get_dark_cal_image( + t_start, + 'nearest', + t_ccd_ref=np.mean(slot_data['TEMPCCD'] - 273.16), + aca_image=False, + ) # all images will be 8x8, with a centered mask, imgrow will always be the one of the 8x8 corner. - imgrow_8x8 = np.where(slot_data['IMGSIZE'] == 8, - slot_data['IMGROW0'], - slot_data['IMGROW0'] - 1 - ) - imgcol_8x8 = np.where(slot_data['IMGSIZE'] == 8, - slot_data['IMGCOL0'], - slot_data['IMGCOL0'] - 1 - ) + imgrow_8x8 = np.where( + slot_data['IMGSIZE'] == 8, slot_data['IMGROW0'], slot_data['IMGROW0'] - 1 + ) + imgcol_8x8 = np.where( + slot_data['IMGSIZE'] == 8, slot_data['IMGCOL0'], slot_data['IMGCOL0'] - 1 + ) # subtract closest dark cal dark = np.zeros([len(slot_data), 8, 8], dtype=np.float64) - staggered_aca_slice(dark_cal.astype(float), dark, 512 + imgrow_8x8, 512 + imgcol_8x8) + staggered_aca_slice( + dark_cal.astype(float), dark, 512 + imgrow_8x8, 512 + imgcol_8x8 + ) img_sub = slot_data['IMGRAW'] - dark * 1.696 / 5 img_sub.mask |= MASK['mouse_bit'] @@ -561,9 +619,13 @@ def get_mag_from_img(slot_data, t_start, ok=True): zag = np.zeros(len(slot_data)) pixel_center = np.arange(8) + 0.5 projected_image = np.ma.sum(slot_data['IMGRAW'], axis=1) - col = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum(projected_image, axis=1) + col = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum( + projected_image, axis=1 + ) projected_image = np.ma.sum(slot_data['IMGRAW'], axis=2) - row = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum(projected_image, axis=1) + row = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum( + projected_image, axis=1 + ) y_pixel = row + imgrow_8x8 z_pixel = col + imgcol_8x8 @@ -574,7 +636,7 @@ def get_mag_from_img(slot_data, t_start, ok=True): 'yang_img': yag, 'zang_img': zag, 'counts_img': img_count, - 'counts_dark': dark_count + 'counts_dark': dark_count, } @@ -583,18 +645,28 @@ def get_mag_from_img(slot_data, t_start, ok=True): 'obsid': 'OBSID at the time the star catalog is commanded (from kadi.commands)', 'slot': 'Slot number', 'type': 'GUI/ACQ/BOT', - 'mp_starcat_time': 'The time at which the star catalog is commanded (from kadi.commands)', - 'tstart': 'Start time of the observation according to kadi.commands (in cxc seconds)', - 'tstop': 'Start time of the observation according to kadi.commands (in cxc seconds)', + 'mp_starcat_time': ( + 'The time at which the star catalog is commanded (from kadi.commands)' + ), + 'tstart': ( + 'Start time of the observation according to kadi.commands (in cxc seconds)' + ), + 'tstop': ( + 'Start time of the observation according to kadi.commands (in cxc seconds)' + ), 'mag_correction': 'Overall correction applied to the magnitude estimate', 'responsivity': 'Responsivity correction applied to the magnitude estimate', 'droop_shift': 'Droop shift correction applied to the magnitude estimate', 'mag_aca': 'ACA star magnitude from the AGASC catalog', 'mag_aca_err': 'ACA star magnitude uncertainty from the AGASC catalog', - 'row': - 'Expected row number, based on star location and yanf/zang from mica.archive.starcheck DB', - 'col': - 'Expected col number, based on star location and yanf/zang from mica.archive.starcheck DB', + 'row': ( + 'Expected row number, based on star location and yanf/zang from' + ' mica.archive.starcheck DB' + ), + 'col': ( + 'Expected col number, based on star location and yanf/zang from' + ' mica.archive.starcheck DB' + ), 'mag_img': 'Magnitude estimate from image telemetry (uncorrected)', 'mag_obs': 'Estimated ACA star magnitude', 'mag_obs_err': 'Estimated ACA star magnitude uncertainty', @@ -605,25 +677,31 @@ def get_mag_from_img(slot_data, t_start, ok=True): 'aoacmag_q75': '3rd quartile of AOACMAG from telemetry', 'counts_img': 'Raw counts from image telemetry, summed over the mouse-bit window', 'counts_dark': 'Expected counts from background, summed over the mouse-bit window', - 'f_kalman': - 'Fraction of all samples where AOACASEQ == "KALM" and AOPCADMD == "NPNT" (n_kalman/n)', + 'f_kalman': ( + 'Fraction of all samples where AOACASEQ == "KALM" and AOPCADMD == "NPNT"' + ' (n_kalman/n)' + ), 'f_track': 'Fraction of kalman samples with AOACFCT == "TRAK" (n_track/n_kalman)', 'f_mag_est_ok': ( - 'Fraction of all samples included in magnitude estimate regardless of centroid residual' - ' (n_mag_est_ok/n_kalman)' + 'Fraction of all samples included in magnitude estimate regardless of centroid' + ' residual (n_mag_est_ok/n_kalman)' + ), + 'f_mag_est_ok_3': ( + 'Fraction of kalman samples with (mag_est_ok & dr3) == True (n_ok_3/n_kalman)' + ), + 'f_mag_est_ok_5': ( + 'Fraction of kalman samples with (mag_est_ok & dbox5) == True (n_ok_5/n_kalman)' ), - 'f_mag_est_ok_3': - 'Fraction of kalman samples with (mag_est_ok & dr3) == True (n_ok_3/n_kalman)', - 'f_mag_est_ok_5': - 'Fraction of kalman samples with (mag_est_ok & dbox5) == True (n_ok_5/n_kalman)', 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5.', # fix this 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", 'f_ok_5': """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all.""", - 'f_dr3': - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec (n_dr3/n_mag_est_ok)', + 'f_dr3': ( + 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' + ' (n_dr3/n_mag_est_ok)' + ), 'f_dbox5': ( 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box ' '(n_dbox5/n_mag_est_ok)' @@ -637,7 +715,9 @@ def get_mag_from_img(slot_data, t_start, ok=True): 'skew': 'Skewness of estimated magnitude', 'kurt': 'Kurtosis of estimated magnitude', 't_mean': 'Mean of estimated magnitude after removing outliers', - 't_mean_err': 'Uncertainty in the mean of estimated magnitude after removing outliers', + 't_mean_err': ( + 'Uncertainty in the mean of estimated magnitude after removing outliers' + ), 't_std': 'Standard deviation of estimated magnitude after removing outliers', 't_skew': 'Skewness of estimated magnitude after removing outliers', 't_kurt': 'Kurtosis of estimated magnitude after removing outliers', @@ -651,7 +731,10 @@ def get_mag_from_img(slot_data, t_start, ok=True): 'dr_star': 'Angle residual', 'obs_ok': 'Boolean flag: everything OK with this observation', 'obs_suspect': 'Boolean flag: this observation is "suspect"', - 'obs_fail': 'Boolean flag: a processing error when estimating magnitude for this observation', + 'obs_fail': ( + 'Boolean flag: a processing error when estimating magnitude for this' + ' observation' + ), 'comments': '', 'w': 'Weight to be used on a weighted mean (1/std)', 'mean_corrected': 'Corrected mean used in weighted mean (t_mean + mag_correction)', @@ -671,8 +754,10 @@ def get_obs_stats(obs, telem=None): :return: dict dictionary with stats """ - logger.debug(f' Getting OBS stats for AGASC ID {obs["agasc_id"]},' - f' OBSID {obs["agasc_id"]} at {obs["mp_starcat_time"]}') + logger.debug( + f' Getting OBS stats for AGASC ID {obs["agasc_id"]},' + f' OBSID {obs["agasc_id"]} at {obs["mp_starcat_time"]}' + ) star_obs_catalogs.load() @@ -680,80 +765,88 @@ def get_obs_stats(obs, telem=None): start = CxoTime(obs['obs_start']).cxcsec stop = CxoTime(obs['obs_stop']).cxcsec - stats = {k: obs[k] for k in - ['agasc_id', 'obsid', 'slot', 'type', 'mp_starcat_time']} + stats = { + k: obs[k] for k in ['agasc_id', 'obsid', 'slot', 'type', 'mp_starcat_time'] + } stats['mp_starcat_time'] = stats['mp_starcat_time'] droop_shift = get_droop_systematic_shift(star['MAG_ACA']) responsivity = get_responsivity(start) - stats.update({'tstart': start, - 'tstop': stop, - 'mag_correction': - responsivity - droop_shift, - 'responsivity': responsivity, - 'droop_shift': droop_shift, - 'mag_aca': star['MAG_ACA'], - 'mag_aca_err': star['MAG_ACA_ERR'] / 100, - 'row': obs['row'], - 'col': obs['col'], - }) + stats.update( + { + 'tstart': start, + 'tstop': stop, + 'mag_correction': -responsivity - droop_shift, + 'responsivity': responsivity, + 'droop_shift': droop_shift, + 'mag_aca': star['MAG_ACA'], + 'mag_aca_err': star['MAG_ACA_ERR'] / 100, + 'row': obs['row'], + 'col': obs['col'], + } + ) # other default values - stats.update({ - 'mag_img': np.inf, - 'mag_obs': np.inf, - 'mag_obs_err': np.inf, - 'aoacmag_mean': np.inf, - 'aoacmag_err': np.inf, - 'aoacmag_q25': np.inf, - 'aoacmag_median': np.inf, - 'aoacmag_q75': np.inf, - 'counts_img': np.inf, - 'counts_dark': np.inf, - 'f_kalman': 0., - 'f_track': 0., - 'f_dbox5': 0., - 'f_dr3': 0., - 'f_mag_est_ok': 0., - 'f_mag_est_ok_3': 0., - 'f_mag_est_ok_5': 0., - 'f_ok': 0., - 'f_ok_3': 0., - 'f_ok_5': 0., - 'q25': np.inf, - 'median': np.inf, - 'q75': np.inf, - 'mean': np.inf, - 'mean_err': np.inf, - 'std': np.inf, - 'skew': np.inf, - 'kurt': np.inf, - 't_mean': np.inf, - 't_mean_err': np.inf, - 't_std': np.inf, - 't_skew': np.inf, - 't_kurt': np.inf, - 'n': 0, - 'n_ok': 0, - 'n_ok_3': 0, - 'n_ok_5': 0, - 'n_mag_est_ok': 0, - 'n_mag_est_ok_3': 0, - 'n_mag_est_ok_5': 0, - 'outliers': -1, - 'lf_variability_100s': np.inf, - 'lf_variability_500s': np.inf, - 'lf_variability_1000s': np.inf, - 'tempccd': np.nan, - 'dr_star': np.inf, - }) + stats.update( + { + 'mag_img': np.inf, + 'mag_obs': np.inf, + 'mag_obs_err': np.inf, + 'aoacmag_mean': np.inf, + 'aoacmag_err': np.inf, + 'aoacmag_q25': np.inf, + 'aoacmag_median': np.inf, + 'aoacmag_q75': np.inf, + 'counts_img': np.inf, + 'counts_dark': np.inf, + 'f_kalman': 0.0, + 'f_track': 0.0, + 'f_dbox5': 0.0, + 'f_dr3': 0.0, + 'f_mag_est_ok': 0.0, + 'f_mag_est_ok_3': 0.0, + 'f_mag_est_ok_5': 0.0, + 'f_ok': 0.0, + 'f_ok_3': 0.0, + 'f_ok_5': 0.0, + 'q25': np.inf, + 'median': np.inf, + 'q75': np.inf, + 'mean': np.inf, + 'mean_err': np.inf, + 'std': np.inf, + 'skew': np.inf, + 'kurt': np.inf, + 't_mean': np.inf, + 't_mean_err': np.inf, + 't_std': np.inf, + 't_skew': np.inf, + 't_kurt': np.inf, + 'n': 0, + 'n_ok': 0, + 'n_ok_3': 0, + 'n_ok_5': 0, + 'n_mag_est_ok': 0, + 'n_mag_est_ok_3': 0, + 'n_mag_est_ok_5': 0, + 'outliers': -1, + 'lf_variability_100s': np.inf, + 'lf_variability_500s': np.inf, + 'lf_variability_1000s': np.inf, + 'tempccd': np.nan, + 'dr_star': np.inf, + } + ) if telem is None: telem = get_telemetry(obs) if len(telem) > 0: stats.update(calc_obs_stats(telem)) - logger.debug(f' slot={stats["slot"]}, f_ok={stats["f_ok"]:.3f}, ' - f'f_mag_est_ok={stats["f_mag_est_ok"]:.3f}, f_dr3={stats["f_dr3"]:.3f}, ' - f'mag={stats["mag_obs"]:.2f}') + logger.debug( + f' slot={stats["slot"]}, f_ok={stats["f_ok"]:.3f}, ' + f'f_mag_est_ok={stats["f_mag_est_ok"]:.3f}, f_dr3={stats["f_dr3"]:.3f}, ' + f'mag={stats["mag_obs"]:.2f}' + ) return stats @@ -778,15 +871,15 @@ def calc_obs_stats(telem): telem = telem[(telem['AOACASEQ'] == 'KALM') & (telem['AOPCADMD'] == 'NPNT')] times = telem['times'] - dr3 = (telem['dr'] < 3) + dr3 = telem['dr'] < 3 dbox5 = (np.abs(telem['dy']) < 5) & (np.abs(telem['dz']) < 5) # Samples used in the magnitude estimation processing # note that SP flag is not included mag_est_ok = (telem['AOACIIR'] == 'OK') & (telem['AOACFCT'] == 'TRAK') - aca_trak = (telem['AOACFCT'] == 'TRAK') - sat_pix = (telem['AOACISP'] == 'OK') - ion_rad = (telem['AOACIIR'] == 'OK') + aca_trak = telem['AOACFCT'] == 'TRAK' + sat_pix = telem['AOACISP'] == 'OK' + ion_rad = telem['AOACIIR'] == 'OK' n_kalman = len(telem) f_kalman = n_kalman / n_total @@ -832,7 +925,9 @@ def calc_obs_stats(telem): if stats['n_mag_est_ok_3'] < 10: return stats - aoacmag_q25, aoacmag_q50, aoacmag_q75 = np.quantile(telem['AOACMAG'][ok], [0.25, 0.5, 0.75]) + aoacmag_q25, aoacmag_q50, aoacmag_q75 = np.quantile( + telem['AOACMAG'][ok], [0.25, 0.5, 0.75] + ) mags = telem['mags'] q25, q50, q75 = np.quantile(mags[ok], [0.25, 0.5, 0.75]) @@ -847,45 +942,51 @@ def calc_obs_stats(telem): s_500s = s_500s[~np.isnan(s_500s)] s_1000s = s_1000s[~np.isnan(s_1000s)] - stats.update({ - 'aoacmag_mean': np.mean(telem['AOACMAG'][ok]), - 'aoacmag_err': np.std(telem['AOACMAG'][ok]), - 'aoacmag_q25': aoacmag_q25, - 'aoacmag_median': aoacmag_q50, - 'aoacmag_q75': aoacmag_q75, - 'q25': q25, - 'median': q50, - 'q75': q75, - 'counts_img': np.mean(telem['counts_img'][ok]), - 'counts_dark': np.mean(telem['counts_dark'][ok]), - 'mean': np.mean(mags[ok]), - 'mean_err': scipy.stats.sem(mags[ok]), - 'std': np.std(mags[ok]), - 'skew': scipy.stats.skew(mags), - 'kurt': scipy.stats.kurtosis(mags), - 't_mean': np.mean(mags[ok & (~outlier)]), - 't_mean_err': scipy.stats.sem(mags[ok & (~outlier)]), - 't_std': np.std(mags[ok & (~outlier)]), - 't_skew': scipy.stats.skew(mags[ok & (~outlier)]), - 't_kurt': scipy.stats.kurtosis(mags[ok & (~outlier)]), - 'outliers': np.count_nonzero(outlier), - 'lf_variability_100s': np.max(s_100s) - np.min(s_100s), - 'lf_variability_500s': np.max(s_500s) - np.min(s_500s), - 'lf_variability_1000s': np.max(s_1000s) - np.min(s_1000s), - 'tempccd': np.mean(telem['TEMPCCD'][ok]) - 273.16, - }) - - stats.update({ - 'mag_img': np.mean(telem['mags_img'][ok & (~outlier)]), - 'mag_obs': stats['t_mean'], - 'mag_obs_err': stats['t_mean_err'] - }) + stats.update( + { + 'aoacmag_mean': np.mean(telem['AOACMAG'][ok]), + 'aoacmag_err': np.std(telem['AOACMAG'][ok]), + 'aoacmag_q25': aoacmag_q25, + 'aoacmag_median': aoacmag_q50, + 'aoacmag_q75': aoacmag_q75, + 'q25': q25, + 'median': q50, + 'q75': q75, + 'counts_img': np.mean(telem['counts_img'][ok]), + 'counts_dark': np.mean(telem['counts_dark'][ok]), + 'mean': np.mean(mags[ok]), + 'mean_err': scipy.stats.sem(mags[ok]), + 'std': np.std(mags[ok]), + 'skew': scipy.stats.skew(mags), + 'kurt': scipy.stats.kurtosis(mags), + 't_mean': np.mean(mags[ok & (~outlier)]), + 't_mean_err': scipy.stats.sem(mags[ok & (~outlier)]), + 't_std': np.std(mags[ok & (~outlier)]), + 't_skew': scipy.stats.skew(mags[ok & (~outlier)]), + 't_kurt': scipy.stats.kurtosis(mags[ok & (~outlier)]), + 'outliers': np.count_nonzero(outlier), + 'lf_variability_100s': np.max(s_100s) - np.min(s_100s), + 'lf_variability_500s': np.max(s_500s) - np.min(s_500s), + 'lf_variability_1000s': np.max(s_1000s) - np.min(s_1000s), + 'tempccd': np.mean(telem['TEMPCCD'][ok]) - 273.16, + } + ) + + stats.update( + { + 'mag_img': np.mean(telem['mags_img'][ok & (~outlier)]), + 'mag_obs': stats['t_mean'], + 'mag_obs_err': stats['t_mean_err'], + } + ) return stats AGASC_ID_STATS_INFO = { - 'last_obs_time': 'CXC seconds corresponding to the last mp_starcat_time for the star', + 'last_obs_time': ( + 'CXC seconds corresponding to the last mp_starcat_time for the star' + ), 'agasc_id': 'AGASC ID of the star', 'mag_aca': 'ACA star magnitude from the AGASC catalog', 'mag_aca_err': 'ACA star magnitude uncertainty from the AGASC catalog', @@ -895,22 +996,27 @@ def calc_obs_stats(telem): 'color': 'Star color from the AGASC catalog', 'n_obsids': 'Number of observations for the star', 'n_obsids_fail': 'Number of observations which give an unexpected error', - 'n_obsids_suspect': - 'Number of observations deemed "suspect" and ignored in the magnitude estimate', + 'n_obsids_suspect': ( + 'Number of observations deemed "suspect" and ignored in the magnitude estimate' + ), 'n_obsids_ok': 'Number of observations considered in the magnitude estimate', 'n_no_mag': 'Number of observations where the star magnitude was not estimated', 'n': 'Total number of image samples for the star', - 'n_mag_est_ok': 'Total number of image samples included in magnitude estimate for the star', + 'n_mag_est_ok': ( + 'Total number of image samples included in magnitude estimate for the star' + ), 'f_mag_est_ok': ( - 'Fraction of all samples included in magnitude estimate regardless of centroid residual' - ' (n_mag_est_ok / n_kalman)' + 'Fraction of all samples included in magnitude estimate regardless of centroid' + ' residual (n_mag_est_ok / n_kalman)' ), 'f_mag_est_ok_3': ( - 'Fraction of kalman samples that are included in magnitude estimate and within 3 arcsec' - ' (n_mag_est_ok_3 / n_kalman)'), + 'Fraction of kalman samples that are included in magnitude estimate and within' + ' 3 arcsec (n_mag_est_ok_3 / n_kalman)' + ), 'f_mag_est_ok_5': ( - 'Fraction of kalman samples that are included in magnitude estimate and within a 5 arcsec' - ' box (n_mag_est_ok_5 / n_kalman)'), + 'Fraction of kalman samples that are included in magnitude estimate and within' + ' a 5 arcsec box (n_mag_est_ok_5 / n_kalman)' + ), 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5.', 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", @@ -922,12 +1028,16 @@ def calc_obs_stats(telem): 'sigma_plus': '84.2% quantile of magnitude over mag-est-ok samples', 'mean': 'Mean of magnitude over mag-est-ok samples', 'std': 'Standard deviation of magnitude over mag-est-ok samples', - 'mag_weighted_mean': - 'Average of magnitudes over observations, weighed by the inverse of its standard deviation', - 'mag_weighted_std': - 'Uncertainty in the weighted magnitude mean', + 'mag_weighted_mean': ( + 'Average of magnitudes over observations, weighed by the inverse of its' + ' standard deviation' + ), + 'mag_weighted_std': 'Uncertainty in the weighted magnitude mean', 't_mean': 'Mean magnitude after removing outliers on a per-observation basis', - 't_std': 'Magnitude standard deviation after removing outliers on a per-observation basis', + 't_std': ( + 'Magnitude standard deviation after removing outliers on a per-observation' + ' basis' + ), 'n_outlier': 'Number of outliers, removed on a per-observation basis', 't_mean_1': 'Mean magnitude after removing 1.5*IQR outliers', 't_std_1': 'Magnitude standard deviation after removing 1.5*IQR outliers', @@ -939,60 +1049,81 @@ def calc_obs_stats(telem): 'selected_rtol': 'abs(mag_obs - mag_aca) > 3 * mag_aca_err', 'selected_mag_aca_err': 'mag_aca_err > 0.2', 'selected_color': '(color == 1.5) | (color == 0.7)', - 't_mean_dr3': + 't_mean_dr3': ( 'Truncated mean magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 't_std_dr3': - 'Truncated magnitude standard deviation after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'mean_dr3': + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 't_std_dr3': ( + 'Truncated magnitude standard deviation after removing outliers and samples' + ' with centroid residual > 3 arcsec on a per-observation basis' + ), + 'mean_dr3': ( 'Mean magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'std_dr3': + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 'std_dr3': ( 'Magnitude standard deviation after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'f_dr3': - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec (n_dr3/n_mag_est_ok)', + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 'f_dr3': ( + 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' + ' (n_dr3/n_mag_est_ok)' + ), 'n_dr3': 'Number of mag-est-ok samples with centroid residual < 3 arcsec', - 'n_dr3_outliers': + 'n_dr3_outliers': ( 'Number of magnitude outliers after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'median_dr3': + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 'median_dr3': ( 'Median magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'sigma_minus_dr3': + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 'sigma_minus_dr3': ( '15.8% quantile of magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'sigma_plus_dr3': + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 'sigma_plus_dr3': ( '84.2% quantile of magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - - 't_mean_dbox5': + 'centroid residual > 3 arcsec on a per-observation basis' + ), + 't_mean_dbox5': ( 'Truncated mean magnitude after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 't_std_dbox5': - 'Truncated magnitude standard deviation after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'mean_dbox5': + 'centroid residual out of a 5 arcsec box, on a per-observation basis' + ), + 't_std_dbox5': ( + 'Truncated magnitude standard deviation after removing outliers and samples' + ' with centroid residual out of a 5 arcsec box, on a per-observation basis' + ), + 'mean_dbox5': ( 'Mean magnitude after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'std_dbox5': + 'centroid residual out of a 5 arcsec box, on a per-observation basis' + ), + 'std_dbox5': ( 'Magnitude standard deviation after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'f_dbox5': 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box', - 'n_dbox5': 'Number of mag-est-ok samples with centroid residual within a 5 arcsec box', - 'n_dbox5_outliers': + 'centroid residual out of a 5 arcsec box, on a per-observation basis' + ), + 'f_dbox5': ( + 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box' + ), + 'n_dbox5': ( + 'Number of mag-est-ok samples with centroid residual within a 5 arcsec box' + ), + 'n_dbox5_outliers': ( 'Number of magnitude outliers after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'median_dbox5': + 'centroid residual out of a 5 arcsec box, on a per-observation basis' + ), + 'median_dbox5': ( 'Median magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis', - 'sigma_minus_dbox5': + 'angular residual out of a 5 arcsec box, on a per-observation basis' + ), + 'sigma_minus_dbox5': ( '15.8% quantile of magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis', - 'sigma_plus_dbox5': + 'angular residual out of a 5 arcsec box, on a per-observation basis' + ), + 'sigma_plus_dbox5': ( '84.2% quantile of magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis', + 'angular residual out of a 5 arcsec box, on a per-observation basis' + ), } @@ -1017,7 +1148,9 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): star_obs_catalogs.load(tstop=tstop) # Get a table of every time the star has been observed - star_obs = star_obs_catalogs.STARS_OBS[star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id] + star_obs = star_obs_catalogs.STARS_OBS[ + star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id + ] if len(star_obs) > 1: star_obs = star_obs.loc['mp_starcat_time', sorted(star_obs['mp_starcat_time'])] @@ -1027,9 +1160,9 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): 'agasc_id': agasc_id, 'mag_aca': np.nan, 'mag_aca_err': np.nan, - 'mag_obs': 0., + 'mag_obs': 0.0, 'mag_obs_err': np.nan, - 'mag_obs_std': 0., + 'mag_obs_std': 0.0, 'color': np.nan, 'n_obsids': 0, 'n_obsids_fail': 0, @@ -1043,7 +1176,7 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): 'n_mag_est_ok': 0, 'n_mag_est_ok_3': 0, 'n_mag_est_ok_5': 0, - 'f_ok': 0., + 'f_ok': 0.0, 'median': 0, 'sigma_minus': 0, 'sigma_plus': 0, @@ -1073,37 +1206,55 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): } for tag in ['dr3', 'dbox5']: - result.update({ - f't_mean_{tag}': 0, - f't_std_{tag}': 0, - f't_mean_{tag}_not': 0, - f't_std_{tag}_not': 0, - f'mean_{tag}': 0, - f'std_{tag}': 0, - f'f_{tag}': 0, - f'n_{tag}': 0, - f'n_{tag}_outliers': 0, - f'median_{tag}': 0, - f'sigma_minus_{tag}': 0, - f'sigma_plus_{tag}': 0, - }) + result.update( + { + f't_mean_{tag}': 0, + f't_std_{tag}': 0, + f't_mean_{tag}_not': 0, + f't_std_{tag}_not': 0, + f'mean_{tag}': 0, + f'std_{tag}': 0, + f'f_{tag}': 0, + f'n_{tag}': 0, + f'n_{tag}_outliers': 0, + f'median_{tag}': 0, + f'sigma_minus_{tag}': 0, + f'sigma_plus_{tag}': 0, + } + ) n_obsids = len(star_obs) # exclude star_obs that are in obs_status_override with status != 0 - excluded_obs = np.array([((oi, ai) in obs_status_override - and obs_status_override[(oi, ai)]['status'] != 0) - for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']]]) + excluded_obs = np.array( + [ + ( + (oi, ai) in obs_status_override + and obs_status_override[(oi, ai)]['status'] != 0 + ) + for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']] + ] + ) if np.any(excluded_obs): - logger.debug(' Excluding observations flagged in obs-status table: ' - f'{list(star_obs[excluded_obs]["obsid"])}') - - included_obs = np.array([((oi, ai) in obs_status_override - and obs_status_override[(oi, ai)]['status'] == 0) - for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']]]) + logger.debug( + ' Excluding observations flagged in obs-status table: ' + f'{list(star_obs[excluded_obs]["obsid"])}' + ) + + included_obs = np.array( + [ + ( + (oi, ai) in obs_status_override + and obs_status_override[(oi, ai)]['status'] == 0 + ) + for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']] + ] + ) if np.any(included_obs): - logger.debug(' Including observations marked OK in obs-status table: ' - f'{list(star_obs[included_obs]["obsid"])}') + logger.debug( + ' Including observations marked OK in obs-status table: ' + f'{list(star_obs[included_obs]["obsid"])}' + ) failures = [] all_telem = [] @@ -1114,51 +1265,62 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): comment = '' if (oi, ai) in obs_status_override: status = obs_status_override[(oi, ai)] - logger.debug(f' overriding status for (AGASC ID {ai}, starcat time {oi}): ' - f'{status["status"]}, {status["comments"]}') + logger.debug( + f' overriding status for (AGASC ID {ai}, starcat time {oi}): ' + f'{status["status"]}, {status["comments"]}' + ) comment = status['comments'] try: last_obs_time = CxoTime(obs['mp_starcat_time']).cxcsec telem = Table(get_telemetry(obs)) obs_stat = get_obs_stats(obs, telem={k: telem[k] for k in telem.colnames}) - obs_stat.update({ - 'obs_ok': ( - included_obs[i] | ( + obs_stat.update( + { + 'obs_ok': included_obs[i] | ( ~excluded_obs[i] & (obs_stat['n'] > 10) & (obs_stat['f_mag_est_ok'] > 0.3) & (obs_stat['lf_variability_100s'] < 1) - ) - ), - 'obs_suspect': False, - 'obs_fail': False, - 'comments': comment - }) + ), + 'obs_suspect': False, + 'obs_fail': False, + 'comments': comment, + } + ) all_telem.append(telem) stats.append(obs_stat) if not obs_stat['obs_ok'] and not excluded_obs[i]: obs_stat['obs_suspect'] = True failures.append( - dict(MagStatsException(msg='Suspect observation', - agasc_id=obs['agasc_id'], - obsid=obs['obsid'], - mp_starcat_time=obs["mp_starcat_time"],))) + dict( + MagStatsException( + msg='Suspect observation', + agasc_id=obs['agasc_id'], + obsid=obs['obsid'], + mp_starcat_time=obs["mp_starcat_time"], + ) + ) + ) except MagStatsException as e: # this except branch deals with exceptions thrown by get_telemetry all_telem.append(None) # length-zero telemetry short-circuits any new call to get_telemetry obs_stat = get_obs_stats(obs, telem=[]) - obs_stat.update({ - 'obs_ok': False, - 'obs_suspect': False, - 'obs_fail': True, - 'comments': comment if excluded_obs[i] else f'Error: {e.msg}.' - }) + obs_stat.update( + { + 'obs_ok': False, + 'obs_suspect': False, + 'obs_fail': True, + 'comments': comment if excluded_obs[i] else f'Error: {e.msg}.', + } + ) stats.append(obs_stat) if not excluded_obs[i]: logger.debug( - f' Error in get_agasc_id_stats({agasc_id=}, obsid={obs["obsid"]}): {e}') + f' Error in get_agasc_id_stats({agasc_id=},' + f' obsid={obs["obsid"]}): {e}' + ) failures.append(dict(e)) stats = Table(stats) @@ -1168,26 +1330,31 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): star = get_star(agasc_id, use_supplement=False) - result.update({ - 'last_obs_time': last_obs_time, - 'mag_aca': star['MAG_ACA'], - 'mag_aca_err': star['MAG_ACA_ERR'] / 100, - 'color': star['COLOR1'], - 'n_obsids_fail': len(failures), - 'n_obsids_suspect': np.count_nonzero(stats['obs_suspect']), - 'n_obsids': n_obsids, - }) + result.update( + { + 'last_obs_time': last_obs_time, + 'mag_aca': star['MAG_ACA'], + 'mag_aca_err': star['MAG_ACA_ERR'] / 100, + 'color': star['COLOR1'], + 'n_obsids_fail': len(failures), + 'n_obsids_suspect': np.count_nonzero(stats['obs_suspect']), + 'n_obsids': n_obsids, + } + ) if not np.any(~excluded_obs): # this happens when all observations have been flagged as not OK a priory (obs-status). - logger.debug(f' Skipping star in get_agasc_id_stats({agasc_id=}).' - ' All observations are flagged as not good.') + logger.debug( + f' Skipping star in get_agasc_id_stats({agasc_id=}).' + ' All observations are flagged as not good.' + ) return result, stats, failures if len(all_telem) - len(failures) <= 0: # and we reach here if some observations were not flagged as bad, but all failed. - logger.debug(f' Error in get_agasc_id_stats({agasc_id=}):' - ' There is no OK observation.') + logger.debug( + f' Error in get_agasc_id_stats({agasc_id=}): There is no OK observation.' + ) return result, stats, failures excluded_obs += np.array([t is None for t in all_telem]) @@ -1197,17 +1364,24 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): if excluded_obs[i]: continue t['obs_ok'] = np.ones_like(t['mag_est_ok'], dtype=bool) * s['obs_ok'] - logger.debug(' identifying outlying observations ' - f'(OBSID={s["obsid"]}, mp_starcat_time={s["mp_starcat_time"]})') + logger.debug( + ' identifying outlying observations ' + f'(OBSID={s["obsid"]}, mp_starcat_time={s["mp_starcat_time"]})' + ) t['obs_outlier'] = np.zeros_like(t['mag_est_ok']) if np.any(t['mag_est_ok']) and s['f_mag_est_ok'] > 0 and s['obs_ok']: iqr = s['q75'] - s['q25'] t['obs_outlier'] = ( t['mag_est_ok'] & (iqr > 0) - & ((t['mags'] < s['q25'] - 1.5 * iqr) | (t['mags'] > s['q75'] + 1.5 * iqr)) + & ( + (t['mags'] < s['q25'] - 1.5 * iqr) + | (t['mags'] > s['q75'] + 1.5 * iqr) + ) ) - all_telem = vstack([Table(t) for i, t in enumerate(all_telem) if not excluded_obs[i]]) + all_telem = vstack( + [Table(t) for i, t in enumerate(all_telem) if not excluded_obs[i]] + ) n_total = len(all_telem) kalman = (all_telem['AOACASEQ'] == 'KALM') & (all_telem['AOPCADMD'] == 'NPNT') @@ -1222,23 +1396,25 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): f_mag_est_ok = np.count_nonzero(mag_est_ok) / len(mag_est_ok) - result.update({ - 'mag_obs_err': min_mag_obs_err, - 'n_obsids_ok': np.count_nonzero(stats['obs_ok']), - 'n_no_mag': ( - np.count_nonzero((~stats['obs_ok'])) - + np.count_nonzero(stats['f_mag_est_ok'][stats['obs_ok']] < 0.3) - ), - 'n': n_total, - 'n_mag_est_ok': np.count_nonzero(mag_est_ok), - 'f_mag_est_ok': f_mag_est_ok, - }) + result.update( + { + 'mag_obs_err': min_mag_obs_err, + 'n_obsids_ok': np.count_nonzero(stats['obs_ok']), + 'n_no_mag': np.count_nonzero((~stats['obs_ok'])) + np.count_nonzero( + stats['f_mag_est_ok'][stats['obs_ok']] < 0.3 + ), + 'n': n_total, + 'n_mag_est_ok': np.count_nonzero(mag_est_ok), + 'f_mag_est_ok': f_mag_est_ok, + } + ) if result['n_mag_est_ok'] < 10: return result, stats, failures - sigma_minus, q25, median, q75, sigma_plus = np.quantile(mags[mag_est_ok], - [0.158, 0.25, 0.5, 0.75, 0.842]) + sigma_minus, q25, median, q75, sigma_plus = np.quantile( + mags[mag_est_ok], [0.158, 0.25, 0.5, 0.75, 0.842] + ) iqr = q75 - q25 outlier_1 = mag_est_ok & ((mags > q75 + 1.5 * iqr) | (mags < q25 - 1.5 * iqr)) outlier_2 = mag_est_ok & ((mags > q75 + 3 * iqr) | (mags < q25 - 3 * iqr)) @@ -1247,43 +1423,49 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): # combine measurements using a weighted mean obs_ok = stats['obs_ok'] min_std = max(0.1, stats[obs_ok]['std'].min()) - stats['w'][obs_ok] = np.where(stats['std'][obs_ok] != 0, - 1. / stats['std'][obs_ok], - 1. / min_std) - stats['mean_corrected'][obs_ok] = stats['t_mean'][obs_ok] + stats['mag_correction'][obs_ok] - stats['weighted_mean'][obs_ok] = stats['mean_corrected'][obs_ok] * stats['w'][obs_ok] - - mag_weighted_mean = (stats[obs_ok]['weighted_mean'].sum() / stats[obs_ok]['w'].sum()) - mag_weighted_std = ( - np.sqrt(((stats[obs_ok]['mean'] - mag_weighted_mean)**2 * stats[obs_ok]['w']).sum() - / stats[obs_ok]['w'].sum()) + stats['w'][obs_ok] = np.where( + stats['std'][obs_ok] != 0, 1.0 / stats['std'][obs_ok], 1.0 / min_std + ) + stats['mean_corrected'][obs_ok] = ( + stats['t_mean'][obs_ok] + stats['mag_correction'][obs_ok] + ) + stats['weighted_mean'][obs_ok] = ( + stats['mean_corrected'][obs_ok] * stats['w'][obs_ok] ) - result.update({ - 'agasc_id': agasc_id, - 'n_mag_est_ok': np.count_nonzero(mag_est_ok), - 'f_mag_est_ok': f_mag_est_ok, - 'median': median, - 'sigma_minus': sigma_minus, - 'sigma_plus': sigma_plus, - 'mean': np.mean(mags[mag_est_ok]), - 'std': np.std(mags[mag_est_ok]), - 'mag_weighted_mean': mag_weighted_mean, - 'mag_weighted_std': mag_weighted_std, - 't_mean': np.mean(mags[mag_est_ok & (~outlier)]), - 't_std': np.std(mags[mag_est_ok & (~outlier)]), - 'n_outlier': np.count_nonzero(mag_est_ok & outlier), - 't_mean_1': np.mean(mags[mag_est_ok & (~outlier_1)]), - 't_std_1': np.std(mags[mag_est_ok & (~outlier_1)]), - 'n_outlier_1': np.count_nonzero(mag_est_ok & outlier_1), - 't_mean_2': np.mean(mags[mag_est_ok & (~outlier_2)]), - 't_std_2': np.std(mags[mag_est_ok & (~outlier_2)]), - 'n_outlier_2': np.count_nonzero(mag_est_ok & outlier_2), - }) + mag_weighted_mean = stats[obs_ok]['weighted_mean'].sum() / stats[obs_ok]['w'].sum() + mag_weighted_std = np.sqrt( + ((stats[obs_ok]['mean'] - mag_weighted_mean) ** 2 * stats[obs_ok]['w']).sum() + / stats[obs_ok]['w'].sum() + ) + + result.update( + { + 'agasc_id': agasc_id, + 'n_mag_est_ok': np.count_nonzero(mag_est_ok), + 'f_mag_est_ok': f_mag_est_ok, + 'median': median, + 'sigma_minus': sigma_minus, + 'sigma_plus': sigma_plus, + 'mean': np.mean(mags[mag_est_ok]), + 'std': np.std(mags[mag_est_ok]), + 'mag_weighted_mean': mag_weighted_mean, + 'mag_weighted_std': mag_weighted_std, + 't_mean': np.mean(mags[mag_est_ok & (~outlier)]), + 't_std': np.std(mags[mag_est_ok & (~outlier)]), + 'n_outlier': np.count_nonzero(mag_est_ok & outlier), + 't_mean_1': np.mean(mags[mag_est_ok & (~outlier_1)]), + 't_std_1': np.std(mags[mag_est_ok & (~outlier_1)]), + 'n_outlier_1': np.count_nonzero(mag_est_ok & outlier_1), + 't_mean_2': np.mean(mags[mag_est_ok & (~outlier_2)]), + 't_std_2': np.std(mags[mag_est_ok & (~outlier_2)]), + 'n_outlier_2': np.count_nonzero(mag_est_ok & outlier_2), + } + ) residual_ok = { 3: all_telem['dr'] < 3, - 5: (np.abs(all_telem['dy']) < 5) & (np.abs(all_telem['dz']) < 5) + 5: (np.abs(all_telem['dy']) < 5) & (np.abs(all_telem['dz']) < 5), } dr_tag = {3: 'dr3', 5: 'dbox5'} for dr in [3, 5]: @@ -1293,46 +1475,65 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): n_ok = np.count_nonzero(aca_trak & sat_pix & ion_rad & residual_ok[dr]) if not np.any(k): continue - sigma_minus, q25, median, q75, sigma_plus = np.quantile(mags[k], - [0.158, 0.25, 0.5, 0.75, 0.842]) + sigma_minus, q25, median, q75, sigma_plus = np.quantile( + mags[k], [0.158, 0.25, 0.5, 0.75, 0.842] + ) outlier = mag_est_ok & all_telem['obs_outlier'] - mag_not = np.nanmean(mags[k2 & (~outlier)]) if np.count_nonzero(k2 & (~outlier)) else np.nan - std_not = np.nanstd(mags[k2 & (~outlier)]) if np.count_nonzero(k2 & (~outlier)) else np.nan - result.update({ - f't_mean_{tag}': np.mean(mags[k & (~outlier)]), - f't_std_{tag}': np.std(mags[k & (~outlier)]), - f't_mean_{tag}_not': mag_not, - f't_std_{tag}_not': std_not, - f'mean_{tag}': np.mean(mags[k]), - f'std_{tag}': np.std(mags[k]), - f'f_{tag}': np.count_nonzero(k) / np.count_nonzero(mag_est_ok), - f'n_{tag}': np.count_nonzero(k), - f'n_{tag}_outliers': np.count_nonzero(k & outlier), - f'f_mag_est_ok_{dr}': np.count_nonzero(k) / len(k), - f'n_mag_est_ok_{dr}': np.count_nonzero(k), - f'median_{tag}': median, - f'sigma_minus_{tag}': sigma_minus, - f'sigma_plus_{tag}': sigma_plus, - f'f_ok_{dr}': (n_ok / n_kalman) if n_kalman else 0, - f'n_ok_{dr}': n_ok, - }) - - result.update({ - 'mag_obs': result['t_mean_dbox5'], - 'mag_obs_err': np.sqrt(result['t_std_dbox5']**2 + min_mag_obs_err**2), - 'mag_obs_std': result['t_std_dbox5'], - 'n_ok': result['n_ok_5'], - 'f_ok': result['f_ok_5'], - }) + mag_not = ( + np.nanmean(mags[k2 & (~outlier)]) + if np.count_nonzero(k2 & (~outlier)) + else np.nan + ) + std_not = ( + np.nanstd(mags[k2 & (~outlier)]) + if np.count_nonzero(k2 & (~outlier)) + else np.nan + ) + result.update( + { + f't_mean_{tag}': np.mean(mags[k & (~outlier)]), + f't_std_{tag}': np.std(mags[k & (~outlier)]), + f't_mean_{tag}_not': mag_not, + f't_std_{tag}_not': std_not, + f'mean_{tag}': np.mean(mags[k]), + f'std_{tag}': np.std(mags[k]), + f'f_{tag}': np.count_nonzero(k) / np.count_nonzero(mag_est_ok), + f'n_{tag}': np.count_nonzero(k), + f'n_{tag}_outliers': np.count_nonzero(k & outlier), + f'f_mag_est_ok_{dr}': np.count_nonzero(k) / len(k), + f'n_mag_est_ok_{dr}': np.count_nonzero(k), + f'median_{tag}': median, + f'sigma_minus_{tag}': sigma_minus, + f'sigma_plus_{tag}': sigma_plus, + f'f_ok_{dr}': (n_ok / n_kalman) if n_kalman else 0, + f'n_ok_{dr}': n_ok, + } + ) + + result.update( + { + 'mag_obs': result['t_mean_dbox5'], + 'mag_obs_err': np.sqrt(result['t_std_dbox5'] ** 2 + min_mag_obs_err**2), + 'mag_obs_std': result['t_std_dbox5'], + 'n_ok': result['n_ok_5'], + 'f_ok': result['f_ok_5'], + } + ) # these are the criteria for including in supplement - result.update({ - 'selected_atol': np.abs(result['mag_obs'] - result['mag_aca']) > 0.3, - 'selected_rtol': np.abs(result['mag_obs'] - result['mag_aca']) > 3 * result['mag_aca_err'], - 'selected_mag_aca_err': result['mag_aca_err'] > 0.2, - 'selected_color': (result['color'] == 1.5) | (np.isclose(result['color'], 0.7)) - }) - - logger.debug(f' stats for AGASC ID {agasc_id}: ' - f' {stats["mag_obs"][0]}') + result.update( + { + 'selected_atol': np.abs(result['mag_obs'] - result['mag_aca']) > 0.3, + 'selected_rtol': ( + np.abs(result['mag_obs'] - result['mag_aca']) + > 3 * result['mag_aca_err'] + ), + 'selected_mag_aca_err': result['mag_aca_err'] > 0.2, + 'selected_color': (result['color'] == 1.5) | ( + np.isclose(result['color'], 0.7) + ), + } + ) + + logger.debug(f' stats for AGASC ID {agasc_id}: {stats["mag_obs"][0]}') return result, stats, failures diff --git a/agasc/supplement/magnitudes/mag_estimate_report.py b/agasc/supplement/magnitudes/mag_estimate_report.py index 56ca1e47..2a474da7 100644 --- a/agasc/supplement/magnitudes/mag_estimate_report.py +++ b/agasc/supplement/magnitudes/mag_estimate_report.py @@ -23,7 +23,7 @@ JINJA2 = jinja2.Environment( loader=jinja2.PackageLoader('agasc.supplement.magnitudes', 'templates'), - autoescape=jinja2.select_autoescape(['html', 'xml']) + autoescape=jinja2.select_autoescape(['html', 'xml']), ) logger = logging.getLogger('agasc.supplement') @@ -52,6 +52,7 @@ class TableEncoder(json.JSONEncoder): ] } """ + def default(self, obj): if isinstance(obj, table.Table): return {name: val.tolist() for name, val in obj.columns.items()} @@ -67,16 +68,19 @@ def default(self, obj): class MagEstimateReport: - def __init__(self, - agasc_stats='mag_stats_agasc.fits', - obs_stats='mag_stats_obsid.fits', - directory='./mag_estimates_reports'): - + def __init__( + self, + agasc_stats='mag_stats_agasc.fits', + obs_stats='mag_stats_obsid.fits', + directory='./mag_estimates_reports', + ): if type(agasc_stats) is table.Table: self.agasc_stats = agasc_stats else: if not Path(agasc_stats).exists: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), agasc_stats) + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), agasc_stats + ) self.agasc_stats = table.Table.read(agasc_stats) self.agasc_stats.convert_bytestring_to_unicode() @@ -84,15 +88,21 @@ def __init__(self, self.obs_stats = obs_stats else: if not Path(obs_stats).exists: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), obs_stats) + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), obs_stats + ) self.obs_stats = table.Table.read(obs_stats) self.obs_stats.convert_bytestring_to_unicode() self.directory = Path(directory) - def single_star_html(self, agasc_id, directory, - static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', - highlight_obs=lambda o: ~o['obs_ok']): + def single_star_html( + self, + agasc_id, + directory, + static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', + highlight_obs=lambda o: ~o['obs_ok'], + ): if np.sum(self.agasc_stats['agasc_id'] == agasc_id) == 0: return @@ -108,34 +118,46 @@ def single_star_html(self, agasc_id, directory, raise Exception(f'agasc_id {agasc_id} has not observations') obs_stat.sort(keys=['mp_starcat_time']) agasc_stat = dict(self.agasc_stats[self.agasc_stats['agasc_id'] == agasc_id][0]) - agasc_stat['n_obs_bad'] = \ - agasc_stat['n_obsids'] - agasc_stat['n_obsids_ok'] - agasc_stat['last_obs'] = ':'.join(obs_stat[-1]['mp_starcat_time'].split(':')[:4]) + agasc_stat['n_obs_bad'] = agasc_stat['n_obsids'] - agasc_stat['n_obsids_ok'] + agasc_stat['last_obs'] = ':'.join( + obs_stat[-1]['mp_starcat_time'].split(':')[:4] + ) # OBSIDs can be repeated obsids = list(np.unique(obs_stat[highlight_obs(obs_stat)]['obsid'])) - args = [{'only_ok': False, 'draw_agasc_mag': True, 'draw_legend': True, 'ylim': 'max'}, - {'title': 'Magnitude Estimates', - 'only_ok': True, - 'ylim': 'stats', - 'highlight_obsid': obsids, - 'draw_obs_mag_stats': True, - 'draw_agasc_mag_stats': True, - 'draw_legend': True, - 'outside_markers': True - }, - {'type': 'flags'}] + args = [ + { + 'only_ok': False, + 'draw_agasc_mag': True, + 'draw_legend': True, + 'ylim': 'max', + }, + { + 'title': 'Magnitude Estimates', + 'only_ok': True, + 'ylim': 'stats', + 'highlight_obsid': obsids, + 'draw_obs_mag_stats': True, + 'draw_agasc_mag_stats': True, + 'draw_legend': True, + 'outside_markers': True, + }, + {'type': 'flags'}, + ] for obsid in obsids: - args.append({'obsid': obsid, - 'ylim': 'fit', - 'only_ok': False, - 'draw_obs_mag_stats': True, - 'draw_agasc_mag_stats': True, - 'draw_legend': True, - 'draw_roll_mean': True, - 'outside_markers': True - }) + args.append( + { + 'obsid': obsid, + 'ylim': 'fit', + 'only_ok': False, + 'draw_obs_mag_stats': True, + 'draw_agasc_mag_stats': True, + 'draw_legend': True, + 'draw_roll_mean': True, + 'outside_markers': True, + } + ) args.append({'type': 'flags', 'obsid': obsid}) fig = self.plot_set(agasc_id, args=args, filename=directory / 'mag_stats.png') plt.close(fig) @@ -146,11 +168,13 @@ def single_star_html(self, agasc_id, directory, "ignore", message="Warning: converting a masked element to nan.", ) - out.write(star_template.render( - agasc_stats=agasc_stat, - obs_stats=obs_stat.as_array(), - static_dir=static_dir, - glossary=GLOSSARY) + out.write( + star_template.render( + agasc_stats=agasc_stat, + obs_stats=obs_stat.as_array(), + static_dir=static_dir, + glossary=GLOSSARY, + ) ) with open(directory / 'data.json', 'w') as json_out: json.dump( @@ -165,15 +189,22 @@ def single_star_html(self, agasc_id, directory, ) return directory / 'index.html' - def multi_star_html(self, sections=None, updated_stars=None, fails=(), - tstart=None, tstop=None, report_date=None, - filename=None, - include_all_stars=False, - make_single_reports=True, - nav_links=None, - highlight_obs=lambda o: ~o['obs_ok'], - static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', - no_progress=None): + def multi_star_html( + self, + sections=None, + updated_stars=None, + fails=(), + tstart=None, + tstop=None, + report_date=None, + filename=None, + include_all_stars=False, + make_single_reports=True, + nav_links=None, + highlight_obs=lambda o: ~o['obs_ok'], + static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', + no_progress=None, + ): if sections is None: sections = [] else: @@ -182,15 +213,26 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(), run_template = JINJA2.get_template('run_report.html') - updated_star_ids = \ - updated_stars['agasc_id'] if updated_stars is not None and len(updated_stars) else [] + updated_star_ids = ( + updated_stars['agasc_id'] + if updated_stars is not None and len(updated_stars) + else [] + ) if updated_stars is None: updated_stars = [] info = { - 'tstart': tstart if tstart else CxoTime(self.obs_stats['mp_starcat_time']).min().date, - 'tstop': tstop if tstop else CxoTime(self.obs_stats['mp_starcat_time']).max().date, - 'report_date': report_date if report_date else CxoTime.now().date + 'tstart': ( + tstart + if tstart + else CxoTime(self.obs_stats['mp_starcat_time']).min().date + ), + 'tstop': ( + tstop + if tstop + else CxoTime(self.obs_stats['mp_starcat_time']).max().date + ), + 'report_date': report_date if report_date else CxoTime.now().date, } if filename is None: @@ -198,70 +240,96 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(), # this is the list of agasc_id for which we will generate individual reports (if possible) if sections: - agasc_ids = np.concatenate([np.array(s['stars'], dtype=int) for s in sections]) + agasc_ids = np.concatenate( + [np.array(s['stars'], dtype=int) for s in sections] + ) else: agasc_ids = [] if include_all_stars: - sections.append({ - 'id': 'other_stars', - 'title': 'Other Stars', - 'stars': self.agasc_stats['agasc_id'][ - ~np.in1d(self.agasc_stats['agasc_id'], agasc_ids)] - }) + sections.append( + { + 'id': 'other_stars', + 'title': 'Other Stars', + 'stars': self.agasc_stats['agasc_id'][ + ~np.in1d(self.agasc_stats['agasc_id'], agasc_ids) + ], + } + ) agasc_ids = self.agasc_stats['agasc_id'] - failed_agasc_ids = [f['agasc_id'] for f in fails - if f['agasc_id'] and int(f['agasc_id']) in self.obs_stats['agasc_id']] + failed_agasc_ids = [ + f['agasc_id'] + for f in fails + if f['agasc_id'] and int(f['agasc_id']) in self.obs_stats['agasc_id'] + ] agasc_ids = np.unique(np.concatenate([agasc_ids, failed_agasc_ids])) # this turns all None into '' in a new list of failures - fails = [{k: '' if v is None else v for k, v in f.items()} for i, f in enumerate(fails)] + fails = [ + {k: '' if v is None else v for k, v in f.items()} + for i, f in enumerate(fails) + ] agasc_stats = self.agasc_stats.copy() # add some extra fields if len(agasc_stats): agasc_stats['n_obs_bad_fail'] = agasc_stats['n_obsids_fail'] - agasc_stats['n_obs_bad'] = agasc_stats['n_obsids'] - agasc_stats['n_obsids_ok'] + agasc_stats['n_obs_bad'] = ( + agasc_stats['n_obsids'] - agasc_stats['n_obsids_ok'] + ) agasc_stats['flag'] = ' ' agasc_stats['flag'][:] = '' - agasc_stats['flag'][(agasc_stats['n_obs_bad'] > 0) - | (agasc_stats['n_obsids'] == 0)] = 'warning' + agasc_stats['flag'][ + (agasc_stats['n_obs_bad'] > 0) | (agasc_stats['n_obsids'] == 0) + ] = 'warning' agasc_stats['flag'][agasc_stats['n_obs_bad_fail'] > 0] = 'danger' - agasc_stats['delta'] = (agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca']) - agasc_stats['sigma'] = ((agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca']) - / agasc_stats['mag_aca_err']) + agasc_stats['delta'] = agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'] + agasc_stats['sigma'] = ( + agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'] + ) / agasc_stats['mag_aca_err'] agasc_stats['new'] = True - agasc_stats['new'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = False + agasc_stats['new'][ + np.in1d(agasc_stats['agasc_id'], updated_star_ids) + ] = False agasc_stats['update_mag_aca'] = np.nan agasc_stats['update_mag_aca_err'] = np.nan agasc_stats['last_obs'] = CxoTime(agasc_stats['last_obs_time']).date if len(updated_stars): - agasc_stats['update_mag_aca'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = \ - updated_stars['mag_aca'] - agasc_stats['update_mag_aca_err'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] =\ - updated_stars['mag_aca_err'] + agasc_stats['update_mag_aca'][ + np.in1d(agasc_stats['agasc_id'], updated_star_ids) + ] = updated_stars['mag_aca'] + agasc_stats['update_mag_aca_err'][ + np.in1d(agasc_stats['agasc_id'], updated_star_ids) + ] = updated_stars['mag_aca_err'] tooltips = { 'warning': 'At least one bad observation', - 'danger': 'At least failed observation' + 'danger': 'At least failed observation', } # make all individual star reports star_reports = {} logger.debug('-' * 80) logger.info("Generating star reports") - for agasc_id in tqdm(np.atleast_1d(agasc_ids).tolist(), - desc='progress', disable=no_progress, unit='star'): + for agasc_id in tqdm( + np.atleast_1d(agasc_ids).tolist(), + desc='progress', + disable=no_progress, + unit='star', + ): try: logger.debug('-' * 80) logger.debug(f'{agasc_id=}') - dirname = self.directory / 'stars' / f'{agasc_id//1e7:03.0f}' / f'{agasc_id:.0f}' + dirname = ( + self.directory + / 'stars' + / f'{agasc_id//1e7:03.0f}' + / f'{agasc_id:.0f}' + ) if make_single_reports: self.single_star_html( - agasc_id, - directory=dirname, - highlight_obs=highlight_obs + agasc_id, directory=dirname, highlight_obs=highlight_obs ) star_reports[agasc_id] = dirname except mag_estimate.MagStatsException: @@ -271,11 +339,14 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(), sections = sections.copy() sections = [section for section in sections if len(section['stars'])] for section in sections: - section['stars'] = agasc_stats[np.in1d(agasc_stats['agasc_id'], - section['stars'])].as_array() + section['stars'] = agasc_stats[ + np.in1d(agasc_stats['agasc_id'], section['stars']) + ].as_array() # this is a hack - star_reports = {i: str(star_reports[i].relative_to(self.directory)) for i in star_reports} + star_reports = { + i: str(star_reports[i].relative_to(self.directory)) for i in star_reports + } # make report if not self.directory.exists(): self.directory.mkdir(parents=True) @@ -285,16 +356,18 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(), "ignore", message="Warning: converting a masked element to nan.", ) - out.write(run_template.render( - info=info, - sections=sections, - failures=fails, - star_reports=star_reports, - nav_links=nav_links, - tooltips=tooltips, - static_dir=static_dir, - glossary=GLOSSARY - )) + out.write( + run_template.render( + info=info, + sections=sections, + failures=fails, + star_reports=star_reports, + nav_links=nav_links, + tooltips=tooltips, + static_dir=static_dir, + glossary=GLOSSARY, + ) + ) json_filename = filename.replace('.html', '.json') if json_filename == filename: @@ -314,20 +387,24 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(), cls=TableEncoder, ) - def plot_agasc_id_single(self, agasc_id, obsid=None, - telem=None, - highlight_obsid=(), - highlight_outliers=True, - only_ok=True, - title=None, - draw_agasc_mag_stats=False, - draw_obs_mag_stats=False, - draw_agasc_mag=False, - draw_roll_mean=False, - draw_legend=False, - ylim='fit', - ax=None, - outside_markers=False): + def plot_agasc_id_single( + self, + agasc_id, + obsid=None, + telem=None, + highlight_obsid=(), + highlight_outliers=True, + only_ok=True, + title=None, + draw_agasc_mag_stats=False, + draw_obs_mag_stats=False, + draw_agasc_mag=False, + draw_roll_mean=False, + draw_legend=False, + ylim='fit', + ax=None, + outside_markers=False, + ): if title is not None: ax.set_title(title) elif obsid is not None: @@ -348,13 +425,17 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, ax = plt.gca() if telem is None: try: - telem = mag_estimate.get_telemetry_by_agasc_id(agasc_id, ignore_exceptions=True) + telem = mag_estimate.get_telemetry_by_agasc_id( + agasc_id, ignore_exceptions=True + ) telem = mag_estimate.add_obs_info(telem, obs_stats) except Exception as e: logger.debug(f'Error making plot: {e}') telem = [] - if len(telem) == 0 or (arg_obsid is not None and np.sum(telem['obsid'] == arg_obsid) == 0): + if len(telem) == 0 or ( + arg_obsid is not None and np.sum(telem['obsid'] == arg_obsid) == 0 + ): msg = 'No Telemetry' if arg_obsid is not None: msg += f' for OBSID {arg_obsid}' @@ -363,7 +444,8 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, np.mean(ax.get_ylim()), msg, horizontalalignment='center', - verticalalignment='center') + verticalalignment='center', + ) return obsids = [arg_obsid] if arg_obsid else np.unique(telem['obsid']) @@ -375,15 +457,22 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, timeline['mag_mean'] = np.nan timeline['mag_std'] = np.nan for i, obsid in enumerate(np.unique(timeline['obsid'])): - sel = (obs_stats['obsid'] == obsid) + sel = obs_stats['obsid'] == obsid if draw_obs_mag_stats and np.any(sel): - timeline['mag_mean'][timeline['obsid'] == obsid] = obs_stats[sel]['mean'][0] - timeline['mag_std'][timeline['obsid'] == obsid] = obs_stats[sel]['std'][0] + timeline['mag_mean'][timeline['obsid'] == obsid] = obs_stats[sel][ + 'mean' + ][0] + timeline['mag_std'][timeline['obsid'] == obsid] = obs_stats[sel]['std'][ + 0 + ] timeline = timeline.as_array() - ok = (telem['AOACASEQ'] == 'KALM') & (telem['AOACIIR'] == 'OK') & \ - (telem['AOPCADMD'] == 'NPNT') + ok = ( + (telem['AOACASEQ'] == 'KALM') + & (telem['AOACIIR'] == 'OK') + & (telem['AOPCADMD'] == 'NPNT') + ) ok = ok & (telem['dr'] < 5) # set the limits of the plot beforehand @@ -393,9 +482,9 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, ok_ylim = ok_ylim & (telem['obsid'] == arg_obsid) if ylim == 'fit': if np.sum(ok_ylim): - q25, q50, q75 = np.quantile(telem['mags'][ok_ylim], [.25, 0.5, 0.75]) + q25, q50, q75 = np.quantile(telem['mags'][ok_ylim], [0.25, 0.5, 0.75]) else: - q25, q50, q75 = np.quantile(telem['mags'], [.25, 0.5, 0.75]) + q25, q50, q75 = np.quantile(telem['mags'], [0.25, 0.5, 0.75]) iqr = max(q75 - q25, 0.05) ax.set_ylim((q25 - 2 * iqr, q75 + 2 * iqr)) ylims_set = True @@ -406,13 +495,17 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, ax.set_ylim((q25 - 3 * iqr, q75 + 3 * iqr)) ylims_set = True elif arg_obsid is None and agasc_stat['mag_obs_std'] > 0: - ylim = (agasc_stat['mag_obs'] - 6 * agasc_stat['mag_obs_std'], - agasc_stat['mag_obs'] + 6 * agasc_stat['mag_obs_std']) + ylim = ( + agasc_stat['mag_obs'] - 6 * agasc_stat['mag_obs_std'], + agasc_stat['mag_obs'] + 6 * agasc_stat['mag_obs_std'], + ) ax.set_ylim(ylim) ylims_set = True if ylim == 'max' or not ylims_set: if np.any(ok_ylim): - ymin, ymax = np.min(telem['mags'][ok_ylim]), np.max(telem['mags'][ok_ylim]) + ymin, ymax = np.min(telem['mags'][ok_ylim]), np.max( + telem['mags'][ok_ylim] + ) else: ymin, ymax = np.min(telem['mags']), np.max(telem['mags']) dy = max(0.3, ymax - ymin) @@ -445,96 +538,132 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, in_obsid = timeline['obsid'] == obsid if len(timeline[timeline['obsid'] == obsid]) == 0: continue - limits[obsid] = (timeline['index'][timeline['obsid'] == obsid].min(), - timeline['index'][timeline['obsid'] == obsid].max()) + limits[obsid] = ( + timeline['index'][timeline['obsid'] == obsid].min(), + timeline['index'][timeline['obsid'] == obsid].max(), + ) if not only_ok and np.any(in_obsid & ~ok): - s = plt.scatter(timeline['index'][in_obsid & ~ok & inside], - timeline[in_obsid & ~ok & inside]['mags'], - s=10, marker='.', color='r', label='not OK') + s = plt.scatter( + timeline['index'][in_obsid & ~ok & inside], + timeline[in_obsid & ~ok & inside]['mags'], + s=10, + marker='.', + color='r', + label='not OK', + ) if i == 0: marker_handles.append(s) _ = plt.scatter( timeline['index'][in_obsid & ~ok & outside_down], bottom[in_obsid & ~ok & outside_down], - s=10, marker='v', color='r' + s=10, + marker='v', + color='r', ) _ = plt.scatter( timeline['index'][in_obsid & ~ok & outside_up], top[in_obsid & ~ok & outside_up], - s=10, marker='^', color='r' + s=10, + marker='^', + color='r', ) if np.any(in_obsid & ok & highlighted): s = plt.scatter( timeline['index'][in_obsid & ok & highlighted & inside], timeline[in_obsid & ok & highlighted & inside]['mags'], - s=10, marker='.', color='orange', label='Highlighted' + s=10, + marker='.', + color='orange', + label='Highlighted', ) if i == 0: marker_handles.append(s) _ = plt.scatter( timeline['index'][in_obsid & ok & highlighted & outside_up], top[in_obsid & ok & highlighted & outside_up], - s=10, marker='^', color='orange', label='Highlighted' + s=10, + marker='^', + color='orange', + label='Highlighted', ) _ = plt.scatter( timeline['index'][in_obsid & ok & highlighted & outside_down], bottom[in_obsid & ok & highlighted & outside_down], - s=10, marker='v', color='orange', label='Highlighted' + s=10, + marker='v', + color='orange', + label='Highlighted', ) if np.any(in_obsid & ok & ~highlighted): s = plt.scatter( timeline['index'][in_obsid & ok & ~highlighted & inside], timeline[in_obsid & ok & ~highlighted & inside]['mags'], - s=10, marker='.', color='k', label='OK' + s=10, + marker='.', + color='k', + label='OK', ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][ - in_obsid & ok & (~highlighted) & outside_down], + timeline['index'][in_obsid & ok & (~highlighted) & outside_down], bottom[in_obsid & ok & ~highlighted & outside_down], - s=10, marker='v', color='k', label='OK' + s=10, + marker='v', + color='k', + label='OK', ) _ = plt.scatter( timeline['index'][in_obsid & ok & ~highlighted & outside_up], top[in_obsid & ok & ~highlighted & outside_up], - s=10, marker='^', color='k', label='OK' + s=10, + marker='^', + color='k', + label='OK', ) - sel = (obs_stats['obsid'] == obsid) + sel = obs_stats['obsid'] == obsid if draw_obs_mag_stats and np.sum(sel): label = '' if i else 'mag$_{OBSID}$' - if (np.isfinite(obs_stats[sel][0]['t_mean']) - and np.isfinite(obs_stats[sel][0]['t_std'])): + if np.isfinite(obs_stats[sel][0]['t_mean']) and np.isfinite( + obs_stats[sel][0]['t_std'] + ): mag_mean = obs_stats[sel][0]['t_mean'] mag_mean_minus = mag_mean - obs_stats[sel][0]['t_std'] mag_mean_plus = mag_mean + obs_stats[sel][0]['t_std'] lh = ax.plot( limits[obsid], [mag_mean, mag_mean], - linewidth=2, color='orange', label=label + linewidth=2, + color='orange', + label=label, ) if i == 0: line_handles += lh - ax.fill_between(limits[obsid], - [mag_mean_minus, mag_mean_minus], - [mag_mean_plus, mag_mean_plus], - color='orange', alpha=0.1, zorder=100) - else: - ( - ax.plot([], [], linewidth=2, color='orange', label=label) + ax.fill_between( + limits[obsid], + [mag_mean_minus, mag_mean_minus], + [mag_mean_plus, mag_mean_plus], + color='orange', + alpha=0.1, + zorder=100, ) + else: + (ax.plot([], [], linewidth=2, color='orange', label=label)) if draw_roll_mean: o = (timeline['obsid'] == obsid) & ok - roll_mean = mag_estimate.rolling_mean(timeline['times'], - timeline['mags'], - window=100, - selection=o) + roll_mean = mag_estimate.rolling_mean( + timeline['times'], timeline['mags'], window=100, selection=o + ) lh = ax.plot( - timeline['index'], roll_mean, '--', - linewidth=1, color='r', label='rolling mean' + timeline['index'], + roll_mean, + '--', + linewidth=1, + color='r', + label='rolling mean', ) if i == 0: line_handles += lh @@ -544,8 +673,13 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, (tmin, tmax) = limits[obsid] ax.plot([tmin, tmin], ax.get_ylim(), ':', color='purple', scaley=False) shift = 0.07 * (ax.get_ylim()[1] - ax.get_ylim()[0]) * (1 + i % 3) - ax.text(np.mean([tmin, tmax]), ax.get_ylim()[0] + shift, f'{obsid}', - verticalalignment='top', horizontalalignment='center') + ax.text( + np.mean([tmin, tmax]), + ax.get_ylim()[0] + shift, + f'{obsid}', + verticalalignment='top', + horizontalalignment='center', + ) if limits: tmax = max([v[1] for v in limits.values()]) ax.plot([tmax, tmax], ax.get_ylim(), ':', color='purple', scaley=False) @@ -555,25 +689,42 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, if draw_agasc_mag: mag_aca = np.mean(agasc_stat['mag_aca']) - line_handles += ( - ax.plot(xlim, [mag_aca, mag_aca], label='mag$_{AGASC}$', - color='green', scalex=False, scaley=False) + line_handles += ax.plot( + xlim, + [mag_aca, mag_aca], + label='mag$_{AGASC}$', + color='green', + scalex=False, + scaley=False, ) - if (draw_agasc_mag_stats - and np.isfinite(agasc_stat['mag_obs']) and agasc_stat['mag_obs'] > 0): + if ( + draw_agasc_mag_stats + and np.isfinite(agasc_stat['mag_obs']) + and agasc_stat['mag_obs'] > 0 + ): mag_weighted_mean = agasc_stat['mag_obs'] mag_weighted_std = agasc_stat['mag_obs_std'] - line_handles += ( - ax.plot(ax.get_xlim(), [mag_weighted_mean, mag_weighted_mean], - label='mag', color='r', scalex=False) + line_handles += ax.plot( + ax.get_xlim(), + [mag_weighted_mean, mag_weighted_mean], + label='mag', + color='r', + scalex=False, + ) + ax.fill_between( + xlim, + [ + mag_weighted_mean - mag_weighted_std, + mag_weighted_mean - mag_weighted_std, + ], + [ + mag_weighted_mean + mag_weighted_std, + mag_weighted_mean + mag_weighted_std, + ], + color='r', + alpha=0.1, ) - ax.fill_between(xlim, - [mag_weighted_mean - mag_weighted_std, - mag_weighted_mean - mag_weighted_std], - [mag_weighted_mean + mag_weighted_std, - mag_weighted_mean + mag_weighted_std], - color='r', alpha=0.1) if draw_legend: ax.set_xlim((xlim[0], xlim[1] + 0.1 * (xlim[1] - xlim[0]))) @@ -590,36 +741,64 @@ def plot_flags(telemetry, ax=None, obsid=None): ax = plt.gca() if len(telemetry) > 0: - timeline = telemetry[['times', 'mags', 'obsid', 'obs_ok', 'dr', 'AOACFCT', - 'AOACASEQ', 'AOACIIR', 'AOPCADMD', - ]] + timeline = telemetry[ + [ + 'times', + 'mags', + 'obsid', + 'obs_ok', + 'dr', + 'AOACFCT', + 'AOACASEQ', + 'AOACIIR', + 'AOPCADMD', + ] + ] timeline['x'] = np.arange(len(timeline)) timeline['y'] = np.ones(len(timeline)) timeline = timeline.as_array() - all_ok = ((timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] == 'TRAK') - & (timeline['AOACIIR'] == 'OK') - & (timeline['dr'] < 3) - ) + all_ok = ( + (timeline['AOACASEQ'] == 'KALM') + & (timeline['AOPCADMD'] == 'NPNT') + & (timeline['AOACFCT'] == 'TRAK') + & (timeline['AOACIIR'] == 'OK') + & (timeline['dr'] < 3) + ) flags = [ - ('dr > 5', ((timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] == 'TRAK') - & (timeline['dr'] >= 5))), + ( + 'dr > 5', + ( + (timeline['AOACASEQ'] == 'KALM') + & (timeline['AOPCADMD'] == 'NPNT') + & (timeline['AOACFCT'] == 'TRAK') + & (timeline['dr'] >= 5) + ), + ), ('Ion. rad.', (timeline['AOACIIR'] != 'OK')), - ('not track', ((timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] != 'TRAK'))), - ('not Kalman', ((timeline['AOACASEQ'] != 'KALM') - | (timeline['AOPCADMD'] != 'NPNT'))), + ( + 'not track', + ( + (timeline['AOACASEQ'] == 'KALM') + & (timeline['AOPCADMD'] == 'NPNT') + & (timeline['AOACFCT'] != 'TRAK') + ), + ), + ( + 'not Kalman', + ( + (timeline['AOACASEQ'] != 'KALM') + | (timeline['AOPCADMD'] != 'NPNT') + ), + ), ] else: timeline_dtype = np.dtype( [(n, float) for n in ['times', 'mags', 'obsid', 'obs_ok', 'dr']] - + [(n, int) for n in ['AOACFCT', 'AOACASEQ', 'AOACIIR', 'AOPCADMD', - 'x', 'y']] + + [ + (n, int) + for n in ['AOACFCT', 'AOACASEQ', 'AOACIIR', 'AOPCADMD', 'x', 'y'] + ] ) timeline = np.array([], dtype=timeline_dtype) all_ok = np.array([], dtype=bool) @@ -645,16 +824,23 @@ def plot_flags(telemetry, ax=None, obsid=None): flags = [('OBS not OK', ~timeline['obs_ok'])] + flags for i, obsid in enumerate(obsids): - limits[obsid] = (timeline['x'][timeline['obsid'] == obsid].min(), - timeline['x'][timeline['obsid'] == obsid].max()) + limits[obsid] = ( + timeline['x'][timeline['obsid'] == obsid].min(), + timeline['x'][timeline['obsid'] == obsid].max(), + ) ok = [f[1] for f in flags] labels = [f[0] for f in flags] ticks = [i for i in range(len(flags))] for i in range(len(ok)): - ax.scatter(timeline['x'][ok[i]], ticks[i] * timeline['y'][ok[i]], s=4, marker='.', - color='k') + ax.scatter( + timeline['x'][ok[i]], + ticks[i] * timeline['y'][ok[i]], + s=4, + marker='.', + color='k', + ) ax.yaxis.set_major_locator(FixedLocator(ticks)) ax.set_yticklabels(labels) ax.set_ylim((-1, ticks[-1] + 1)) @@ -668,32 +854,42 @@ def plot_flags(telemetry, ax=None, obsid=None): ax.axvline(tmax, linestyle=':', color='purple') divider = make_axes_locatable(ax) - ax_dr = divider.append_axes("bottom", size='25%', pad=0., sharex=ax) + ax_dr = divider.append_axes("bottom", size='25%', pad=0.0, sharex=ax) if len(timeline) > 0: dr = timeline['dr'].copy() dr[dr > 10] = 10 ax_dr.scatter( - timeline['x'][all_ok & (dr < 10)], dr[all_ok & (dr < 10)], - s=3, marker='.', color='k' + timeline['x'][all_ok & (dr < 10)], + dr[all_ok & (dr < 10)], + s=3, + marker='.', + color='k', ) ax_dr.scatter( - timeline['x'][all_ok & (dr >= 10)], dr[all_ok & (dr >= 10)], - s=3, marker='^', color='k' + timeline['x'][all_ok & (dr >= 10)], + dr[all_ok & (dr >= 10)], + s=3, + marker='^', + color='k', ) ax_dr.scatter( - timeline['x'][~all_ok & (dr < 10)], dr[~all_ok & (dr < 10)], - s=3, marker='.', color='r' + timeline['x'][~all_ok & (dr < 10)], + dr[~all_ok & (dr < 10)], + s=3, + marker='.', + color='r', ) ax_dr.scatter( timeline['x'][~all_ok & (dr >= 10)], dr[~all_ok & (dr >= 10)], - s=3, marker='^', - color='r' + s=3, + marker='^', + color='r', ) ax_dr.set_ylabel('dr') ax_dr.set_ylim((-0.5, 10.5)) - ax_dr.set_yticks([0., 2.5, 5, 7.5, 10], minor=True) - ax_dr.set_yticks([0., 5, 10], minor=False) + ax_dr.set_yticks([0.0, 2.5, 5, 7.5, 10], minor=True) + ax_dr.set_yticks([0.0, 5, 10], minor=False) ax_dr.grid(True, axis='y', linestyle=':') for i, obsid in enumerate(sorted_obsids): @@ -708,10 +904,11 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): if telem is None: try: - telem = mag_estimate.get_telemetry_by_agasc_id(agasc_id, ignore_exceptions=True) + telem = mag_estimate.get_telemetry_by_agasc_id( + agasc_id, ignore_exceptions=True + ) telem = mag_estimate.add_obs_info( - telem, - self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + telem, self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] ) except Exception as e: logger.debug(f'Error making plot: {e}') @@ -742,17 +939,19 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): return fig -def email_bad_obs_report(bad_obs, to, - sender=f"{getpass.getuser()}@{platform.uname()[1]}"): +def email_bad_obs_report( + bad_obs, to, sender=f"{getpass.getuser()}@{platform.uname()[1]}" +): date = CxoTime().date[:14] message = JINJA2.get_template('email_report.txt') msg = MIMEText(message) msg["From"] = sender msg["To"] = to - msg["Subject"] = \ - f"{len(bad_obs)} suspicious observation{'s' if len(bad_obs) else ''}" \ + msg["Subject"] = ( + f"{len(bad_obs)} suspicious observation{'s' if len(bad_obs) else ''}" f" in magnitude estimates at {date}." + ) p = Popen(["/usr/sbin/sendmail", "-t", "-oi"], stdin=PIPE) p.communicate(msg.as_string().encode()) @@ -763,8 +962,10 @@ def email_bad_obs_report(bad_obs, to, could be a null 4x4 image when the ACA is not tracking. Be aware that the sample period is faster for null 4x4 images (1.025 sec) than for typical 6x6 or 8x8 tracked images (2.05 or 4.1 sec respectively).""", - 'Kalman samples': - 'Subset of samples when ACA could be tracking stars (AOPCADMD == NPNT & AOACASEQ == KALM)', + 'Kalman samples': ( + 'Subset of samples when ACA could be tracking stars (AOPCADMD == NPNT &' + ' AOACASEQ == KALM)' + ), 'dr3': """ Subset of tracked Kalman samples with radial centroid residual < 3 arcsec. This corresponds to "high quality" readouts. This effectively includes the @@ -774,11 +975,16 @@ def email_bad_obs_report(bad_obs, to, This corresponds to spatial filter used by the OBC for inclusion in Kalman filter. This effectively includes the track subset (AOACFCT == TRAK) because readouts with no TRAK are assigned an infinite centroid residual.""", - 'track': 'Subset of Kalman samples where the image is being tracked (AOACFCT == TRAK)', + 'track': ( + 'Subset of Kalman samples where the image is being tracked (AOACFCT == TRAK)' + ), 'sat_pix': 'Subset of Kalman samples with saturated pixel flag OK (AOACISP == OK)', - 'ion_rad': 'Subset of Kalman samples with ionizing radiation flag OK (AOACIIR == OK)', - 'mag_est_ok': - "Subset of Kalman samples that have a magnitude estimate (track & ion_rad)", + 'ion_rad': ( + 'Subset of Kalman samples with ionizing radiation flag OK (AOACIIR == OK)' + ), + 'mag_est_ok': ( + "Subset of Kalman samples that have a magnitude estimate (track & ion_rad)" + ), 'n_total': 'Total number of sample regardless of OBC PCAD status', 'n': 'Synonym for n_total', 'n_kalman': 'Number of Kalman samples', @@ -790,14 +996,18 @@ def email_bad_obs_report(bad_obs, to, 'n_mag_est_ok': 'Number of (track & ion_rad) samples', 'n_mag_est_ok_3': 'Number of (track & ion_rad & dr3) samples', 'n_mag_est_ok_5': 'Number of (track & ion_rad & dbox5) samples', - 'f_dr3': + 'f_dr3': ( 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' - '((mag_est_ok & n_dr3)/n_mag_est_ok)', - 'f_dbox5': + '((mag_est_ok & n_dr3)/n_mag_est_ok)' + ), + 'f_dbox5': ( 'Fraction of mag-est-ok samples with centroid within 5 arcsec box' - '((mag_est_ok & n_dbox5)/n_mag_est_ok)', - 'f_mag_est_ok': """n_mag_est_ok_3/n_kalman. This is a measure of the fraction of time during - an observation that a magnitude estimate is available.""", + '((mag_est_ok & n_dbox5)/n_mag_est_ok)' + ), + 'f_mag_est_ok': ( + """n_mag_est_ok_3/n_kalman. This is a measure of the fraction of time during + an observation that a magnitude estimate is available.""" + ), 'f_mag_est_ok_3': "n_mag_est_ok_3/n_kalman.", 'f_mag_est_ok_5': "n_mag_est_ok_5/n_kalman.", 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5', diff --git a/agasc/supplement/magnitudes/star_obs_catalogs.py b/agasc/supplement/magnitudes/star_obs_catalogs.py index 2f5245bc..997e7750 100644 --- a/agasc/supplement/magnitudes/star_obs_catalogs.py +++ b/agasc/supplement/magnitudes/star_obs_catalogs.py @@ -26,11 +26,15 @@ def get_star_observations(start=None, stop=None, obsid=None): observations = observations[~observations['starcat_date'].mask] # the following line removes manual commands observations = observations[observations['source'] != 'CMD_EVT'] - catalogs = commands.get_starcats_as_table(start=start, stop=stop, obsid=obsid, unique=True) + catalogs = commands.get_starcats_as_table( + start=start, stop=stop, obsid=obsid, unique=True + ) catalogs = catalogs[np.in1d(catalogs['type'], ['BOT', 'GUI'])] star_obs = join(observations, catalogs, keys=join_keys) star_obs.rename_columns(['id', 'starcat_date'], ['agasc_id', 'mp_starcat_time']) - star_obs['row'], star_obs['col'] = yagzag_to_pixels(star_obs['yang'], star_obs['zang']) + star_obs['row'], star_obs['col'] = yagzag_to_pixels( + star_obs['yang'], star_obs['zang'] + ) # Add mag_aca_err column filename = os.path.join(os.environ['SKA'], 'data', 'agasc', 'proseco_agasc_1p7.h5') diff --git a/agasc/supplement/magnitudes/update_mag_supplement.py b/agasc/supplement/magnitudes/update_mag_supplement.py index c029a7b9..651d942e 100755 --- a/agasc/supplement/magnitudes/update_mag_supplement.py +++ b/agasc/supplement/magnitudes/update_mag_supplement.py @@ -17,7 +17,11 @@ from astropy import time, units as u from mica.starcheck import get_starcheck_catalog -from agasc.supplement.magnitudes import star_obs_catalogs, mag_estimate, mag_estimate_report as msr +from agasc.supplement.magnitudes import ( + star_obs_catalogs, + mag_estimate, + mag_estimate_report as msr, +) from agasc.supplement.utils import save_version, MAGS_DTYPE from cxotime import CxoTime @@ -33,6 +37,7 @@ def level0_archive_time_range(): """ import sqlite3 import os + db_file = os.path.expandvars('$SKA/data/mica/archive/aca0/archfiles.db3') with sqlite3.connect(db_file) as connection: cursor = connection.cursor() @@ -69,10 +74,9 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres try: logger.debug('-' * 80) logger.debug(f'{agasc_id=}') - agasc_stat, obs_stat, obs_fail = \ - mag_estimate.get_agasc_id_stats(agasc_id=agasc_id, - obs_status_override=obs_status_override, - tstop=tstop) + agasc_stat, obs_stat, obs_fail = mag_estimate.get_agasc_id_stats( + agasc_id=agasc_id, obs_status_override=obs_status_override, tstop=tstop + ) agasc_stats.append(agasc_stat) obs_stats.append(obs_stat) fails += obs_fail @@ -84,7 +88,9 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres # transform Exception to MagStatsException for standard book keeping msg = f'Unexpected Error: {e}' logger.debug(msg) - fails.append(dict(mag_estimate.MagStatsException(agasc_id=agasc_id, msg=msg))) + fails.append( + dict(mag_estimate.MagStatsException(agasc_id=agasc_id, msg=msg)) + ) exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is not None: trace = traceback.format_exception(exc_type, exc_value, exc_traceback) @@ -103,13 +109,19 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres agasc_stats = None obs_stats = None # transform Exception to MagStatsException for standard book keeping - fails.append(dict(mag_estimate.MagStatsException( - msg=f'Exception at end of get_agasc_id_stats: {str(e)}'))) + fails.append( + dict( + mag_estimate.MagStatsException( + msg=f'Exception at end of get_agasc_id_stats: {str(e)}' + ) + ) + ) return obs_stats, agasc_stats, fails -def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, tstop=None, - no_progress=None): +def get_agasc_id_stats_pool( + agasc_ids, obs_status_override=None, batch_size=100, tstop=None, no_progress=None +): """ Call update_mag_stats.get_agasc_id_stats multiple times using a multiprocessing.Pool @@ -135,11 +147,14 @@ def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, finished = 0 logger.info(f'Processing {batch_size} stars per job') for i in range(0, len(agasc_ids), batch_size): - args.append(agasc_ids[i:i + batch_size]) + args.append(agasc_ids[i : i + batch_size]) with Pool() as pool: for arg in args: - jobs.append(pool.apply_async(get_agasc_id_stats, - [arg, obs_status_override, tstop, True])) + jobs.append( + pool.apply_async( + get_agasc_id_stats, [arg, obs_status_override, tstop, True] + ) + ) bar = tqdm(total=len(jobs), desc='progress', disable=no_progress, unit='job') while finished < len(jobs): finished = sum([f.ready() for f in jobs]) @@ -156,9 +171,13 @@ def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, job.get() except Exception as e: for agasc_id in arg: - fails.append(dict( - mag_estimate.MagStatsException(agasc_id=agasc_id, msg=f'Failed job: {e}') - )) + fails.append( + dict( + mag_estimate.MagStatsException( + agasc_id=agasc_id, msg=f'Failed job: {e}' + ) + ) + ) results = [job.get() for job in jobs if job.successful()] @@ -181,9 +200,9 @@ def _update_table(table_old, table_new, keys): ) table_old = table_old.copy() new_row = np.ones(len(table_new), dtype=bool) - _, i_new, i_old = np.intersect1d(table_new[keys].as_array(), - table_old[keys].as_array(), - return_indices=True) + _, i_new, i_old = np.intersect1d( + table_new[keys].as_array(), table_old[keys].as_array(), return_indices=True + ) new_row[i_new] = False columns = table_old.as_array().dtype.names table_old[i_old] = table_new[i_new][columns] @@ -209,19 +228,23 @@ def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): filename = outdir / 'mag_stats_agasc.fits' logger.debug(f'Updating {filename}') if filename.exists(): - agasc_stats = _update_table(table.Table.read(filename), agasc_stats, - keys=['agasc_id']) + agasc_stats = _update_table( + table.Table.read(filename), agasc_stats, keys=['agasc_id'] + ) os.remove(filename) for column in agasc_stats.colnames: if column in mag_estimate.AGASC_ID_STATS_INFO: - agasc_stats[column].description = mag_estimate.AGASC_ID_STATS_INFO[column] + agasc_stats[column].description = mag_estimate.AGASC_ID_STATS_INFO[ + column + ] agasc_stats.write(filename) if obs_stats is not None and len(obs_stats): filename = outdir / 'mag_stats_obsid.fits' logger.debug(f'Updating {filename}') if filename.exists(): - obs_stats = _update_table(table.Table.read(filename), obs_stats, - keys=['agasc_id', 'obsid']) + obs_stats = _update_table( + table.Table.read(filename), obs_stats, keys=['agasc_id', 'obsid'] + ) os.remove(filename) for column in obs_stats.colnames: if column in mag_estimate.OBS_STATS_INFO: @@ -250,16 +273,16 @@ def update_supplement(agasc_stats, filename, include_all=True): return [], [] if include_all: - outliers_new = agasc_stats[ - (agasc_stats['n_obsids_ok'] > 0) - ] + outliers_new = agasc_stats[(agasc_stats['n_obsids_ok'] > 0)] else: outliers_new = agasc_stats[ (agasc_stats['n_obsids_ok'] > 0) - & (agasc_stats['selected_atol'] - | agasc_stats['selected_rtol'] - | agasc_stats['selected_color'] - | agasc_stats['selected_mag_aca_err']) + & ( + agasc_stats['selected_atol'] + | agasc_stats['selected_rtol'] + | agasc_stats['selected_color'] + | agasc_stats['selected_mag_aca_err'] + ) ] outliers_new['mag_aca'] = outliers_new['mag_obs'] outliers_new['mag_aca_err'] = outliers_new['mag_obs_err'] @@ -277,9 +300,11 @@ def update_supplement(agasc_stats, filename, include_all=True): if 'mags' in h5.root: outliers_current = h5.root.mags[:] # find the indices of agasc_ids in both current and new lists - _, i_new, i_cur = np.intersect1d(outliers_new['agasc_id'], - outliers_current['agasc_id'], - return_indices=True) + _, i_new, i_cur = np.intersect1d( + outliers_new['agasc_id'], + outliers_current['agasc_id'], + return_indices=True, + ) current = outliers_current[i_cur] new = outliers_new[i_new] @@ -287,19 +312,25 @@ def update_supplement(agasc_stats, filename, include_all=True): i_cur = i_cur[current['last_obs_time'] != new['last_obs_time']] i_new = i_new[current['last_obs_time'] != new['last_obs_time']] # overwrite current values with new values (and calculate diff to return) - updated_stars = np.zeros(len(outliers_new[i_new]), - dtype=MAGS_DTYPE) - updated_stars['mag_aca'] = (outliers_new[i_new]['mag_aca'] - - outliers_current[i_cur]['mag_aca']) - updated_stars['mag_aca_err'] = (outliers_new[i_new]['mag_aca_err'] - - outliers_current[i_cur]['mag_aca_err']) + updated_stars = np.zeros(len(outliers_new[i_new]), dtype=MAGS_DTYPE) + updated_stars['mag_aca'] = ( + outliers_new[i_new]['mag_aca'] - outliers_current[i_cur]['mag_aca'] + ) + updated_stars['mag_aca_err'] = ( + outliers_new[i_new]['mag_aca_err'] + - outliers_current[i_cur]['mag_aca_err'] + ) updated_stars['agasc_id'] = outliers_new[i_new]['agasc_id'] outliers_current[i_cur] = outliers_new[i_new] # find agasc_ids in new list but not in current list - new_stars = ~np.in1d(outliers_new['agasc_id'], outliers_current['agasc_id']) + new_stars = ~np.in1d( + outliers_new['agasc_id'], outliers_current['agasc_id'] + ) # and add them to the current list - outliers_current = np.concatenate([outliers_current, outliers_new[new_stars]]) + outliers_current = np.concatenate( + [outliers_current, outliers_new[new_stars]] + ) outliers = np.sort(outliers_current) new_stars = outliers_new[new_stars]['agasc_id'] @@ -328,27 +359,34 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): for mp_starcat_time in mp_starcat_times: rows = obs_stats[obs_stats['mp_starcat_time'] == mp_starcat_time] rows.sort(keys='agasc_id') - obs.append({ - 'mp_starcat_time': mp_starcat_time, - 'obsid': obs_stats['obsid'], - 'agasc_id': list(rows['agasc_id']), - 'status': 1, - 'comments': obs_stats['comment'] - }) + obs.append( + { + 'mp_starcat_time': mp_starcat_time, + 'obsid': obs_stats['obsid'], + 'agasc_id': list(rows['agasc_id']), + 'status': 1, + 'comments': obs_stats['comment'], + } + ) for fail in fails: if fail['agasc_id'] is None or fail['mp_starcat_time'] is None: continue - mp_starcat_times = fail['mp_starcat_time'] if type(fail['mp_starcat_time']) is list \ + mp_starcat_times = ( + fail['mp_starcat_time'] + if type(fail['mp_starcat_time']) is list else [fail['mp_starcat_time']] + ) agasc_id = fail['agasc_id'] for mp_starcat_time in mp_starcat_times: - obs.append({ - 'mp_starcat_time': mp_starcat_time, - 'obsid': fail['obsid'], - 'agasc_id': [agasc_id], - 'status': 1, - 'comments': fail['msg'] - }) + obs.append( + { + 'mp_starcat_time': mp_starcat_time, + 'obsid': fail['obsid'], + 'agasc_id': [agasc_id], + 'status': 1, + 'comments': fail['msg'], + } + ) if len(obs) == 0: if filename and filename.exists(): filename.unlink() @@ -360,7 +398,9 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): if cat: cat = cat['cat'] maxmags = dict(zip(cat['id'], cat['maxmag'])) - agasc_ids += [(agasc_id, maxmags.get(agasc_id, -1)) for agasc_id in o['agasc_id']] + agasc_ids += [ + (agasc_id, maxmags.get(agasc_id, -1)) for agasc_id in o['agasc_id'] + ] else: agasc_ids += [(agasc_id, -1) for agasc_id in obs['agasc_id']] @@ -423,19 +463,20 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): return result -def do(start, - stop, - output_dir, - agasc_ids=None, - report=False, - reports_dir=None, - report_date=None, - multi_process=False, - include_bad=False, - dry_run=False, - no_progress=None, - email='', - ): +def do( + start, + stop, + output_dir, + agasc_ids=None, + report=False, + reports_dir=None, + report_date=None, + multi_process=False, + include_bad=False, + dry_run=False, + no_progress=None, + email='', +): """ :param start: cxotime.CxoTime @@ -477,8 +518,9 @@ def do(start, skip = True if agasc_ids is None: - obs_in_time = ((star_obs_catalogs.STARS_OBS['mp_starcat_time'] >= start) - & (star_obs_catalogs.STARS_OBS['mp_starcat_time'] <= stop)) + obs_in_time = (star_obs_catalogs.STARS_OBS['mp_starcat_time'] >= start) & ( + star_obs_catalogs.STARS_OBS['mp_starcat_time'] <= stop + ) agasc_ids = sorted(star_obs_catalogs.STARS_OBS[obs_in_time]['agasc_id']) else: agasc_ids = np.intersect1d(agasc_ids, star_obs_catalogs.STARS_OBS['agasc_id']) @@ -500,46 +542,64 @@ def do(start, with tables.File(filename, 'r') as h5: if not include_bad and 'bad' in h5.root: logger.info('Excluding bad stars') - stars_obs = stars_obs[~np.in1d(stars_obs['agasc_id'], h5.root.bad[:]['agasc_id'])] + stars_obs = stars_obs[ + ~np.in1d(stars_obs['agasc_id'], h5.root.bad[:]['agasc_id']) + ] if 'obs' in h5.root: obs_status_override = table.Table(h5.root.obs[:]) obs_status_override.convert_bytestring_to_unicode() obs_status_override = { - (r['mp_starcat_time'], r['agasc_id']): - {'status': r['status'], 'comments': r['comments']} + (r['mp_starcat_time'], r['agasc_id']): { + 'status': r['status'], + 'comments': r['comments'], + } for r in obs_status_override } if 'mags' in h5.root and len(stars_obs): outliers_current = h5.root.mags[:] - times = stars_obs[['agasc_id', 'mp_starcat_time']].group_by( - 'agasc_id').groups.aggregate(lambda d: np.max(CxoTime(d)).date) + times = ( + stars_obs[['agasc_id', 'mp_starcat_time']] + .group_by('agasc_id') + .groups.aggregate(lambda d: np.max(CxoTime(d)).date) + ) if len(outliers_current): - times = table.join(times, - table.Table(outliers_current[['agasc_id', 'last_obs_time']]), - join_type='left') + times = table.join( + times, + table.Table(outliers_current[['agasc_id', 'last_obs_time']]), + join_type='left', + ) else: times['last_obs_time'] = table.MaskedColumn( np.zeros(len(times), dtype=h5.root.mags.dtype['last_obs_time']), - mask=np.ones(len(times), dtype=bool) + mask=np.ones(len(times), dtype=bool), ) if skip: if hasattr(times['last_obs_time'], 'mask'): # the mask exists if there are stars in stars_obs # that are not in outliers_current - update = (times['last_obs_time'].mask - | ((~times['last_obs_time'].mask) - & (CxoTime(times['mp_starcat_time']).cxcsec - > times['last_obs_time']).data) - ) + update = times['last_obs_time'].mask | ( + (~times['last_obs_time'].mask) + & ( + CxoTime(times['mp_starcat_time']).cxcsec + > times['last_obs_time'] + ).data + ) else: - update = (CxoTime(times['mp_starcat_time']).cxcsec > times['last_obs_time']) - - stars_obs = stars_obs[np.in1d(stars_obs['agasc_id'], times[update]['agasc_id'])] + update = ( + CxoTime(times['mp_starcat_time']).cxcsec + > times['last_obs_time'] + ) + + stars_obs = stars_obs[ + np.in1d(stars_obs['agasc_id'], times[update]['agasc_id']) + ] agasc_ids = np.sort(np.unique(stars_obs['agasc_id'])) if len(update) - np.sum(update): - logger.info(f'Skipping {len(update) - np.sum(update)} ' - f'stars already in the supplement') + logger.info( + f'Skipping {len(update) - np.sum(update)} ' + 'stars already in the supplement' + ) if len(stars_obs) == 0: logger.info(f'There are no new observations to process') @@ -551,16 +611,18 @@ def do(start, if dry_run: return - obs_stats, agasc_stats, fails = \ - get_stats(agasc_ids, tstop=stop, - obs_status_override=obs_status_override, - no_progress=no_progress) + obs_stats, agasc_stats, fails = get_stats( + agasc_ids, + tstop=stop, + obs_status_override=obs_status_override, + no_progress=no_progress, + ) failed_global = [f for f in fails if not f['agasc_id'] and not f['obsid']] failed_stars = [f for f in fails if f['agasc_id'] and not f['obsid']] failed_obs = [f for f in fails if f['obsid']] msg = ( - f'Got:\n' + 'Got:\n' f' {0 if obs_stats is None else len(obs_stats)} OBSIDs,' f' {0 if agasc_stats is None else len(agasc_stats)} stars,' ) @@ -579,7 +641,9 @@ def do(start, obs_status_file = output_dir / 'obs_status.yml' try: - write_obs_status_yaml([], fails=failed_obs + failed_stars, filename=obs_status_file) + write_obs_status_yaml( + [], fails=failed_obs + failed_stars, filename=obs_status_file + ) except Exception as e: logger.warning(f'Failed to write {obs_status_file}: {e}') @@ -613,7 +677,7 @@ def do(start, nav_links = { 'previous': f'../{(report_date - week).date[:8]}/index.html', 'up': '..', - 'next': f'../{(report_date + week).date[:8]}/index.html' + 'next': f'../{(report_date + week).date[:8]}/index.html', } # If the report data file exists, the arguments for the report from the file are @@ -632,22 +696,25 @@ def do(start, multi_star_html_args['no_progress'] = no_progress else: - sections = [{ - 'id': 'new_stars', - 'title': 'New Stars', - 'stars': new_stars - }, { - 'id': 'updated_stars', - 'title': 'Updated Stars', - 'stars': updated_stars['agasc_id'] if len(updated_stars) else [] - }, { - 'id': 'other_stars', - 'title': 'Other Stars', - 'stars': list(agasc_stats['agasc_id'][ - ~np.in1d(agasc_stats['agasc_id'], new_stars) - & ~np.in1d(agasc_stats['agasc_id'], updated_stars['agasc_id']) - ]) - } + sections = [ + {'id': 'new_stars', 'title': 'New Stars', 'stars': new_stars}, + { + 'id': 'updated_stars', + 'title': 'Updated Stars', + 'stars': updated_stars['agasc_id'] if len(updated_stars) else [], + }, + { + 'id': 'other_stars', + 'title': 'Other Stars', + 'stars': list( + agasc_stats['agasc_id'][ + ~np.in1d(agasc_stats['agasc_id'], new_stars) + & ~np.in1d( + agasc_stats['agasc_id'], updated_stars['agasc_id'] + ) + ] + ), + }, ] multi_star_html_args = dict( @@ -660,14 +727,14 @@ def do(start, tstop=stop, nav_links=nav_links, include_all_stars=False, - no_progress=no_progress + no_progress=no_progress, ) try: report = msr.MagEstimateReport( agasc_stats=output_dir / 'mag_stats_agasc.fits', obs_stats=output_dir / 'mag_stats_obsid.fits', - directory=report_dir + directory=report_dir, ) report.multi_star_html(**multi_star_html_args) latest = reports_dir / 'latest' @@ -690,10 +757,7 @@ def do(start, report_data_file = report_dir / report_data_file.name if not report_dir.exists(): report_dir.mkdir(parents=True) - report_data = { - 'args': multi_star_html_args, - 'directory': report_dir - } + report_data = {'args': multi_star_html_args, 'directory': report_dir} with open(report_data_file, 'wb') as fh: pickle.dump(report_data, fh) logger.info(f'Report data saved in {report_data_file}') diff --git a/agasc/supplement/utils.py b/agasc/supplement/utils.py index 719c7dd3..7c1a0ec9 100644 --- a/agasc/supplement/utils.py +++ b/agasc/supplement/utils.py @@ -12,8 +12,13 @@ from ..paths import SUPPLEMENT_FILENAME, default_agasc_dir -__all__ = ['get_supplement_table', 'save_version', - 'update_mags_table', 'update_obs_table', 'add_bad_star'] +__all__ = [ + 'get_supplement_table', + 'save_version', + 'update_mags_table', + 'update_obs_table', + 'add_bad_star', +] logger = logging.getLogger('agasc.supplement') @@ -22,25 +27,26 @@ AGASC_SUPPLEMENT_TABLES = ('mags', 'bad', 'obs', 'last_updated', 'agasc_versions') -BAD_DTYPE = np.dtype([ - ('agasc_id', np.int32), - ('source', np.int16) -]) +BAD_DTYPE = np.dtype([('agasc_id', np.int32), ('source', np.int16)]) -MAGS_DTYPE = np.dtype([ - ('agasc_id', np.int32), - ('mag_aca', np.float32), - ('mag_aca_err', np.float32), - ('last_obs_time', np.float64) -]) +MAGS_DTYPE = np.dtype( + [ + ('agasc_id', np.int32), + ('mag_aca', np.float32), + ('mag_aca_err', np.float32), + ('last_obs_time', np.float64), + ] +) -OBS_DTYPE = np.dtype([ - ('mp_starcat_time', ' 0 else 1 - bad = [[agasc_id, default if source is None else source] - for agasc_id, source in bad] + bad = [ + [agasc_id, default if source is None else source] for agasc_id, source in bad + ] bad_star_ids, bad_star_source = np.array(bad).astype(int).T @@ -314,7 +342,9 @@ def add_bad_star(filename, bad, dry_run=False, create=False): logger.info('') logger.info('IMPORTANT:') logger.info('Edit following if source ID is new:') - logger.info(' https://github.com/sot/agasc/wiki/Add-bad-star-to-AGASC-supplement-manually') + logger.info( + ' https://github.com/sot/agasc/wiki/Add-bad-star-to-AGASC-supplement-manually' + ) logger.info('') logger.info('The wiki page also includes instructions for test, review, approval') logger.info('and installation.') @@ -347,10 +377,15 @@ def update_obs_table(filename, obs, dry_run=False, create=False): Create a supplement file if it does not exist """ - update_table(filename, obs, 'obs', OBS_DTYPE, - keys=['agasc_id', 'mp_starcat_time'], - dry_run=dry_run, - create=create) + update_table( + filename, + obs, + 'obs', + OBS_DTYPE, + keys=['agasc_id', 'mp_starcat_time'], + dry_run=dry_run, + create=create, + ) def update_mags_table(filename, mags, dry_run=False, create=False): @@ -370,7 +405,12 @@ def update_mags_table(filename, mags, dry_run=False, create=False): :param dry_run: bool Do not save the table. """ - update_table(filename, mags, 'mags', MAGS_DTYPE, - keys=['agasc_id'], - dry_run=dry_run, - create=create) + update_table( + filename, + mags, + 'mags', + MAGS_DTYPE, + keys=['agasc_id'], + dry_run=dry_run, + create=create, + ) diff --git a/agasc/tests/test_agasc_1.py b/agasc/tests/test_agasc_1.py index b4b1be81..e630ca34 100644 --- a/agasc/tests/test_agasc_1.py +++ b/agasc/tests/test_agasc_1.py @@ -13,37 +13,44 @@ def test_multi_agasc(): - tempdir = tempfile.mkdtemp() # Make two custom agasc files from the miniagasc, using 20 stars from # around the middle of the table with tables.open_file(agasc.default_agasc_file()) as h5: middle = int(len(h5.root.data) // 2) - stars1 = Table(h5.root.data[middle: middle + 20]) + stars1 = Table(h5.root.data[middle : middle + 20]) stars1.write(os.path.join(tempdir, 'stars1.h5'), path='data') - stars2 = Table(h5.root.data[middle + 20:middle + 60]) + stars2 = Table(h5.root.data[middle + 20 : middle + 60]) stars2.write(os.path.join(tempdir, 'stars2.h5'), path='data') # Fetch all the stars from a custom agasc and make sure we have the right number of stars # with no errors - all_stars2 = agasc.get_agasc_cone(0, 90, radius=180, - agasc_file=os.path.join(tempdir, 'stars2.h5')) + all_stars2 = agasc.get_agasc_cone( + 0, 90, radius=180, agasc_file=os.path.join(tempdir, 'stars2.h5') + ) assert len(all_stars2) == len(stars2) # Fetch all the stars from the other custom agasc and do the same. The point of the two files # is to confirm that the caching behavior in agasc doesn't cause problems with fetches - all_stars1 = agasc.get_agasc_cone(0, 90, radius=180, - agasc_file=os.path.join(tempdir, 'stars1.h5')) + all_stars1 = agasc.get_agasc_cone( + 0, 90, radius=180, agasc_file=os.path.join(tempdir, 'stars1.h5') + ) assert len(all_stars1) == len(stars1) # Do a position filtered search using the first star in the table as a reference and make sure # we get the same star from the reference agasc. Do this with the stars2 file as this confirms # that we can switch back and forth between files and get the correct content. - cone2 = agasc.get_agasc_cone(all_stars2['RA'][0], all_stars2['DEC'][0], radius=0.000001, - agasc_file=os.path.join(tempdir, 'stars2.h5')) + cone2 = agasc.get_agasc_cone( + all_stars2['RA'][0], + all_stars2['DEC'][0], + radius=0.000001, + agasc_file=os.path.join(tempdir, 'stars2.h5'), + ) # And this is a read of the default agasc file after the custom ones so should confirm that # the custom files didn't break that access. - cone2_full = agasc.get_agasc_cone(all_stars2['RA'][0], all_stars2['DEC'][0], radius=0.000001) + cone2_full = agasc.get_agasc_cone( + all_stars2['RA'][0], all_stars2['DEC'][0], radius=0.000001 + ) assert cone2[0]['AGASC_ID'] == cone2_full[0]['AGASC_ID'] # Confirm that there is just one star in this test setup (not a module test, but confirms test # setup is as intended). diff --git a/agasc/tests/test_agasc_2.py b/agasc/tests/test_agasc_2.py index 53c6b2ff..b7cfe58e 100644 --- a/agasc/tests/test_agasc_2.py +++ b/agasc/tests/test_agasc_2.py @@ -38,7 +38,9 @@ try: ascrc_file = '{}/.ascrc'.format(os.environ['HOME']) assert os.path.exists(ascrc_file) - ascds_env = Ska.Shell.getenv('source {} -r release'.format(ascrc_file), shell='tcsh') + ascds_env = Ska.Shell.getenv( + 'source {} -r release'.format(ascrc_file), shell='tcsh' + ) assert 'ASCDS_BIN' in ascds_env cmd = 'mp_get_agasc -r 10 -d 20 -w 0.01' # Run the command to check for bad status (which will throw exception) @@ -153,7 +155,7 @@ def get_reference_agasc_values(ra, dec, version='1p7'): return dat -RAS = np.hstack([0., 180., 0.1, 180., 275.36]) +RAS = np.hstack([0.0, 180.0, 0.1, 180.0, 275.36]) DECS = np.hstack([89.9, -89.9, 0.0, 0.0, 8.09]) # The (275.36, 8.09) coordinate fails unless date=2000:001 due to # mp_get_agasc not accounting for proper motion. @@ -176,7 +178,7 @@ def test_agasc_conesearch(ra, dec, version): radius=TEST_RADIUS, agasc_file=agasc.get_agasc_filename('miniagasc_*', version=version), date='2000:001', - fix_color1=False + fix_color1=False, ) test_file = get_test_file(ra, dec, version) print(f'\nWriting {test_file} based on miniagasc\n') @@ -199,9 +201,14 @@ def test_against_ds_agasc(ra, dec): def _test_agasc(ra, dec, ref_stars, version='1p7'): agasc_file = agasc.get_agasc_filename('miniagasc_*', version=version) - stars1 = agasc.get_agasc_cone(ra, dec, radius=TEST_RADIUS, - agasc_file=agasc_file, - date='2000:001', fix_color1=False) + stars1 = agasc.get_agasc_cone( + ra, + dec, + radius=TEST_RADIUS, + agasc_file=agasc_file, + date='2000:001', + fix_color1=False, + ) stars1.sort('AGASC_ID') stars2 = ref_stars.copy() @@ -220,8 +227,7 @@ def _test_agasc(ra, dec, ref_stars, version='1p7'): # Second make sure that the non-common stars are all just at the edge # of the faint mag limit, due to precision loss in mp_get_agasc - for s1, s2 in ((stars1, stars2), - (stars2, stars1)): + for s1, s2 in ((stars1, stars2), (stars2, stars1)): mm1 = set(s1['AGASC_ID']) - set(s2['AGASC_ID']) for agasc_id in mm1: idx = np.flatnonzero(s1['AGASC_ID'] == agasc_id)[0] @@ -278,10 +284,13 @@ def test_get_stars3(): def test_get_stars_many(): """Test get_stars() with at least GET_STARS_METHOD_THRESHOLD (5000) stars""" from .. import agasc + stars = agasc.get_agasc_cone(0, 0, radius=0.5) agasc_ids = stars['AGASC_ID'] stars1 = agasc.get_stars(agasc_ids, dates='2020:001') # read_where method - stars2 = agasc.get_stars(agasc_ids, dates='2020:001', method_threshold=1) # read entire AGASC + stars2 = agasc.get_stars( + agasc_ids, dates='2020:001', method_threshold=1 + ) # read entire AGASC assert stars1.get_stars_method == 'tables_read_where' assert stars2.get_stars_method == 'read_entire_agasc' @@ -292,7 +301,7 @@ def test_get_stars_many(): def test_float16(): - stars = agasc.get_agasc_cone(np.float16(219.90279), np.float16(-60.83358), .015) + stars = agasc.get_agasc_cone(np.float16(219.90279), np.float16(-60.83358), 0.015) assert stars['AGASC_ID'][0] == 1180612176 @@ -310,23 +319,36 @@ def test_proper_motion(): stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2017:001') assert len(stars) == 0 - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2017:001', - pm_filter=False) + stars = agasc.get_agasc_cone( + star['RA'], star['DEC'], radius, date='2017:001', pm_filter=False + ) assert len(stars) == 1 - stars = agasc.get_agasc_cone(star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001') + stars = agasc.get_agasc_cone( + star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001' + ) assert len(stars) == 1 - stars = agasc.get_agasc_cone(star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001', - pm_filter=False) + stars = agasc.get_agasc_cone( + star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001', pm_filter=False + ) assert len(stars) == 0 @pytest.mark.parametrize( "agasc_id,date,ra_pmcorr,dec_pmcorr,label", - [(1180612288, '2020:001', 219.864331, -60.831868, "high proper motion, epoch 2000"), - (198451217, '2020:001', 247.892206, 19.276605, "epoch 1982 star"), - (501219465, '2020:001', 166.998976, 52.822080, "epoch 1984 star")]) + [ + ( + 1180612288, + '2020:001', + 219.864331, + -60.831868, + "high proper motion, epoch 2000", + ), + (198451217, '2020:001', 247.892206, 19.276605, "epoch 1982 star"), + (501219465, '2020:001', 166.998976, 52.822080, "epoch 1984 star"), + ], +) def test_add_pmcorr_is_consistent(agasc_id, date, ra_pmcorr, dec_pmcorr, label): """ Check that the proper-motion corrected position is consistent reference/regress values. @@ -352,8 +374,9 @@ def test_agasc_id(ra, dec, radius=0.2, nstar_limit=5): agasc_file = agasc.get_agasc_filename("miniagasc_*", version=DS_AGASC_VERSION) print('ra, dec =', ra, dec) - stars = agasc.get_agasc_cone(ra, dec, radius=radius, agasc_file=agasc_file, - fix_color1=False) + stars = agasc.get_agasc_cone( + ra, dec, radius=radius, agasc_file=agasc_file, fix_color1=False + ) stars.sort('AGASC_ID') for agasc_id in stars['AGASC_ID'][:nstar_limit]: @@ -372,10 +395,10 @@ def test_proseco_agasc_1p7(): mini_file = agasc.get_agasc_filename("miniagasc_*", version="1p7") # Stars looking toward galactic center (dense!) - p_stars = agasc.get_agasc_cone(-266, -29, 3, - agasc_file=proseco_file, date='2000:001') - m_stars = agasc.get_agasc_cone(-266, -29, 3, - agasc_file=mini_file, date='2000:001') + p_stars = agasc.get_agasc_cone( + -266, -29, 3, agasc_file=proseco_file, date='2000:001' + ) + m_stars = agasc.get_agasc_cone(-266, -29, 3, agasc_file=mini_file, date='2000:001') # Every miniagasc_1p7 star is in proseco_agasc_1p7 m_ids = m_stars['AGASC_ID'] @@ -545,7 +568,13 @@ def test_get_supplement_table_mags_dict(): def test_get_supplement_table_obs(): obs = agasc.get_supplement_table('obs') assert isinstance(obs, Table) - assert obs.colnames == ['mp_starcat_time', 'agasc_id', 'obsid', 'status', 'comments'] + assert obs.colnames == [ + 'mp_starcat_time', + 'agasc_id', + 'obsid', + 'status', + 'comments', + ] @pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason='no obs in supplement') diff --git a/agasc/tests/test_obs_status.py b/agasc/tests/test_obs_status.py index a9998e0e..55282a11 100644 --- a/agasc/tests/test_obs_status.py +++ b/agasc/tests/test_obs_status.py @@ -17,48 +17,89 @@ # star_obs_catalogs.STARS_OBS is used to determine all AGASC IDs in an observation # or the last observation time of a given AGASC_ID. # this array is used to monkey-patch star_obs_catalogs.STARS_OBS -STARS_OBS = np.array([ - (56314, 114950168, '2010:110:14:57:43.442'), (56314, 114950584, '2010:110:14:57:43.442'), - (56314, 114952056, '2010:110:14:57:43.442'), (56314, 114952792, '2010:110:14:57:43.442'), - (56314, 114952824, '2010:110:14:57:43.442'), (56314, 114955056, '2010:110:14:57:43.442'), - (56314, 114956608, '2010:110:14:57:43.442'), (56314, 115347520, '2010:110:14:57:43.442'), - (56312, 357045496, '2010:110:16:59:51.399'), (56312, 357049064, '2010:110:16:59:51.399'), - (56312, 357051640, '2010:110:16:59:51.399'), (56312, 357054680, '2010:110:16:59:51.399'), - (56312, 358220224, '2010:110:16:59:51.399'), (56312, 358222256, '2010:110:16:59:51.399'), - (56312, 358224400, '2010:110:16:59:51.399'), (56312, 358757768, '2010:110:16:59:51.399'), - (56313, 441853632, '2010:110:15:07:49.876'), (56313, 441854760, '2010:110:15:07:49.876'), - (56313, 441855776, '2010:110:15:07:49.876'), (56313, 441856032, '2010:110:15:07:49.876'), - (56313, 441856400, '2010:110:15:07:49.876'), (56313, 441980072, '2010:110:15:07:49.876'), - (56313, 491391592, '2010:110:15:07:49.876'), (56313, 491394504, '2010:110:15:07:49.876'), - (56311, 563087864, '2010:110:18:59:46.600'), (56311, 563088952, '2010:110:18:59:46.600'), - (56311, 563089432, '2010:110:18:59:46.600'), (56311, 563091784, '2010:110:18:59:46.600'), - (56311, 563092520, '2010:110:18:59:46.600'), (56311, 563612488, '2010:110:18:59:46.600'), - (56311, 563612792, '2010:110:18:59:46.600'), (56311, 563617352, '2010:110:18:59:46.600'), - (56310, 624826320, '2010:110:20:33:54.789'), (56310, 624826464, '2010:110:20:33:54.789'), - (56310, 624828488, '2010:110:20:33:54.789'), (56310, 624831328, '2010:110:20:33:54.789'), - (56310, 624831392, '2010:110:20:33:54.789'), (56310, 624954248, '2010:110:20:33:54.789'), - (56310, 624956216, '2010:110:20:33:54.789'), (56310, 625476960, '2010:110:20:33:54.789'), - (12203, 697581832, '2010:111:10:30:46.876'), (12203, 697963056, '2010:111:10:30:46.876'), - (12203, 697963288, '2010:111:10:30:46.876'), (12203, 697970576, '2010:111:10:30:46.876'), - (12203, 697973824, '2010:111:10:30:46.876'), (56308, 732697144, '2010:110:23:23:49.708'), - (56308, 732698416, '2010:110:23:23:49.708'), (56309, 762184312, '2010:110:22:02:30.780'), - (56309, 762184768, '2010:110:22:02:30.780'), (56309, 762185584, '2010:110:22:02:30.780'), - (56309, 762186016, '2010:110:22:02:30.780'), (56309, 762186080, '2010:110:22:02:30.780'), - (56309, 762191224, '2010:110:22:02:30.780'), (56309, 762579584, '2010:110:22:02:30.780'), - (56309, 762581024, '2010:110:22:02:30.780'), (56308, 806748432, '2010:110:23:23:49.708'), - (56308, 806748880, '2010:110:23:23:49.708'), (56308, 806750112, '2010:110:23:23:49.708'), - (56308, 806750408, '2010:110:23:23:49.708'), (56308, 806750912, '2010:110:23:23:49.708'), - (56308, 806751424, '2010:110:23:23:49.708'), (56306, 956708808, '2010:111:02:18:49.052'), - (56306, 957219128, '2010:111:02:18:49.052'), (56306, 957221200, '2010:111:02:18:49.052'), - (56306, 957222432, '2010:111:02:18:49.052'), (56306, 957229080, '2010:111:02:18:49.052'), - (56306, 957230976, '2010:111:02:18:49.052'), (56306, 957233920, '2010:111:02:18:49.052'), - (56306, 957369088, '2010:111:02:18:49.052'), (11849, 1019347720, '2010:111:12:26:54.536'), - (11849, 1019348536, '2010:111:12:26:54.536'), (11849, 1019350904, '2010:111:12:26:54.536'), - (11849, 1019354232, '2010:111:12:26:54.536'), (11849, 1019357032, '2010:111:12:26:54.536'), - (11980, 1198184872, '2010:111:03:21:11.299'), (11980, 1198190648, '2010:111:03:21:11.299'), - (11980, 1198190664, '2010:111:03:21:11.299'), (11980, 1198191400, '2010:111:03:21:11.299'), - (11980, 1198192456, '2010:111:03:21:11.299')], - dtype=[('obsid', ' 0, 'Table.write was never called' @@ -577,31 +929,102 @@ def mock_write(*args, **kwargs): assert False if 'path' in kwargs and kwargs['path'] == 'obs': mock_write.n_calls += 1 - ref = table.Table(np.array([ - (56311, 563087864, 1, '', '2010:110:18:59:46.600'), - (56311, 563088952, 1, '', '2010:110:18:59:46.600'), - (56311, 563089432, 1, '', '2010:110:18:59:46.600'), - (56311, 563091784, 1, '', '2010:110:18:59:46.600'), - (56311, 563092520, 1, '', '2010:110:18:59:46.600'), - (56311, 563612488, 1, '', '2010:110:18:59:46.600'), - (56311, 563612792, 1, '', '2010:110:18:59:46.600'), - (56311, 563617352, 1, '', '2010:110:18:59:46.600'), - (56308, 806750112, 0, '', '2010:110:23:23:49.708'), - (11849, 1019348536, 1, 'just removed them', '2010:111:12:26:54.536'), - (11849, 1019350904, 1, 'just removed them', '2010:111:12:26:54.536'), - (56314, 114950168, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114950584, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114952056, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114952792, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114952824, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114955056, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114956608, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 115347520, 1, 'removed because I felt like it', '2010:110:14:57:43.442')], - dtype=[('obsid', ' 0, "Table.write was never called" def test_override(monkeypatch): _monkeypatch_star_obs_catalogs_( - monkeypatch, - test_file=TEST_DATA_DIR / 'mag-stats.h5', - path='/obs_status' + monkeypatch, test_file=TEST_DATA_DIR / 'mag-stats.h5', path='/obs_status' ) telem = _monkeypatch_get_telemetry_( - monkeypatch, - test_file=TEST_DATA_DIR / 'mag-stats.h5', - path='/obs_status/telem' + monkeypatch, test_file=TEST_DATA_DIR / 'mag-stats.h5', path='/obs_status/telem' ) # Case 1. There are two previously unknown suspect observations out of 5. @@ -762,14 +1226,15 @@ def test_override(monkeypatch): assert len(obs_stats) == 5 assert agasc_stats['n_obsids_fail'] == 2 assert agasc_stats['n_obsids_ok'] == 3 - assert agasc_stats['n_obsids_suspect'] == 2 # two suspect count as "fail" in this context + assert ( + agasc_stats['n_obsids_suspect'] == 2 + ) # two suspect count as "fail" in this context assert not np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), agasc_stats['mag_obs'] ) assert np.isclose( np.mean(telem[np.in1d(telem['obsid'], [12800, 23682, 23683])]['mags']), - agasc_stats['mag_obs'] + agasc_stats['mag_obs'], ) # Case 2. Four observations (including the suspect) are marked with non-zero status @@ -778,22 +1243,24 @@ def test_override(monkeypatch): ('2018:296:15:53:14.596', 10492752): {'status': 1, 'comments': ''}, ('2021:015:00:01:45.585', 10492752): {'status': 1, 'comments': ''}, ('2021:089:02:48:00.575', 10492752): {'status': 1, 'comments': ''}, - ('2021:201:02:58:03.250', 10492752): {'status': 1, 'comments': ''} + ('2021:201:02:58:03.250', 10492752): {'status': 1, 'comments': ''}, } - agasc_stats, obs_stats, fails = \ - mag_estimate.get_agasc_id_stats(10492752, obs_status_override=obs_status_override) + agasc_stats, obs_stats, fails = mag_estimate.get_agasc_id_stats( + 10492752, obs_status_override=obs_status_override + ) assert len(fails) == 0 assert agasc_stats['n_obsids_fail'] == 0 assert agasc_stats['n_obsids_ok'] == 1 - assert agasc_stats['n_obsids_suspect'] == 0 # no fails because all status==1 skipped + assert ( + agasc_stats['n_obsids_suspect'] == 0 + ) # no fails because all status==1 skipped assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), agasc_stats['mag_obs'] ) assert not np.isclose( np.mean(telem[np.in1d(telem['obsid'], [12800, 23682, 23683])]['mags']), - agasc_stats['mag_obs'] + agasc_stats['mag_obs'], ) # Case 3: @@ -803,8 +1270,9 @@ def test_override(monkeypatch): ('2021:015:00:01:45.585', 10492752): {'status': 0, 'comments': ''}, ('2021:089:02:48:00.575', 10492752): {'status': 1, 'comments': ''}, } - agasc_stats, obs_stats, fails = \ - mag_estimate.get_agasc_id_stats(10492752, obs_status_override=obs_status_override) + agasc_stats, obs_stats, fails = mag_estimate.get_agasc_id_stats( + 10492752, obs_status_override=obs_status_override + ) assert len(fails) == 1 assert agasc_stats['n_obsids_fail'] == 1 @@ -812,7 +1280,7 @@ def test_override(monkeypatch): assert agasc_stats['n_obsids_suspect'] == 1 # one failed assert np.isclose( np.mean(telem[np.in1d(telem['obsid'], [12800, 23681, 23683])]['mags']), - agasc_stats['mag_obs'] + agasc_stats['mag_obs'], ) @@ -851,6 +1319,7 @@ def get_telemetry(obs): if obs['obsid'] in telem['obsid']: return telem[telem['obsid'] == obs['obsid']] raise Exception(f'{obsid=} not in test telemetry') + monkeypatch.setattr(mag_estimate, 'get_telemetry', get_telemetry) return telem @@ -890,7 +1359,7 @@ def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): path='/obs_status/cat/STARS_OBS', serialize_meta=True, append=True, - overwrite=True + overwrite=True, ) telem = mag_estimate.get_telemetry_by_agasc_id(10492752) @@ -899,12 +1368,12 @@ def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): # might be in maneuver mode or in acquisition. These times come from the kadi events v1 version, # but they do not matter much. telem_by_obsid = [ - telem[(telem['obsid'] == 12800) & (telem['times'] > 435047672.)][:100], + telem[(telem['obsid'] == 12800) & (telem['times'] > 435047672.0)][:100], # only 10 points, excluding the beginning - telem[(telem['obsid'] == 23681) & (telem['times'] > 727057549.)][:10], - telem[(telem['obsid'] == 23682) & (telem['times'] > 733462165.)][:100], - telem[(telem['obsid'] == 23683) & (telem['times'] > 743139160.)][:100], - telem[(telem['obsid'] == 48900) & (telem['times'] > 656698074.)][:100], + telem[(telem['obsid'] == 23681) & (telem['times'] > 727057549.0)][:10], + telem[(telem['obsid'] == 23682) & (telem['times'] > 733462165.0)][:100], + telem[(telem['obsid'] == 23683) & (telem['times'] > 743139160.0)][:100], + telem[(telem['obsid'] == 48900) & (telem['times'] > 656698074.0)][:100], ] telem_by_obsid[-1]['mags_img'] += 0.01 * np.exp(np.arange(100) / 20) telem_by_obsid[-1]['mags'] += 0.01 * np.exp(np.arange(100) / 20) @@ -917,6 +1386,7 @@ def recreate_test_supplement(supplement_filename=TEST_DATA_DIR / 'agasc_suppleme # this is not a test function, but a function to generate the test supplement from scratch # whenever it needs updating, so all the data is actually contained in this file from _pytest.monkeypatch import MonkeyPatch + monkeypatch = MonkeyPatch() monkeypatch.setitem(__builtins__, 'open', _open) monkeypatch.setattr(star_obs_catalogs, 'STARS_OBS', STARS_OBS) @@ -928,16 +1398,14 @@ def recreate_test_supplement(supplement_filename=TEST_DATA_DIR / 'agasc_suppleme status = update_supplement.parse_args(filename='file_0.yml') print('obs') print(status['obs']) - update_supplement.update_obs_table(supplement_filename, - status['obs'], - dry_run=False) + update_supplement.update_obs_table( + supplement_filename, status['obs'], dry_run=False + ) print('mags') print(status['mags']) - update_supplement.update_mags_table(supplement_filename, - status['mags'], - dry_run=False) + update_supplement.update_mags_table( + supplement_filename, status['mags'], dry_run=False + ) print('bad') print(status['bad']) - update_supplement.add_bad_star(supplement_filename, - status['bad'], - dry_run=False) + update_supplement.add_bad_star(supplement_filename, status['bad'], dry_run=False) diff --git a/create_derived_agasc_h5.py b/create_derived_agasc_h5.py index 57a3e2fb..449c87ec 100644 --- a/create_derived_agasc_h5.py +++ b/create_derived_agasc_h5.py @@ -44,8 +44,6 @@ from agasc import default_agasc_dir - - def get_parser(): parser = argparse.ArgumentParser( description="Create derived AGASC HDF5 file", usage=__doc__ @@ -69,7 +67,7 @@ def get_parser(): parser.add_argument( "--filter-faint", action="store_true", - help=('Filter: stars["MAG_ACA"] - 3.0 * stars["MAG_ACA_ERR"] / 100.0 < 11.5 '), + help='Filter: stars["MAG_ACA"] - 3.0 * stars["MAG_ACA_ERR"] / 100.0 < 11.5 ', ) parser.add_argument( "--proseco-columns", diff --git a/create_near_neighbor_ids.py b/create_near_neighbor_ids.py index c41f2b99..7721159d 100644 --- a/create_near_neighbor_ids.py +++ b/create_near_neighbor_ids.py @@ -81,9 +81,7 @@ def main(): if id != sp["AGASC_ID"]: near_ids.add(id) - outfile = ( - Path(args.outdir) / f"{agasc_full.name[:-3]}_near_neighbor_ids.fits.gz" - ) + outfile = Path(args.outdir) / f"{agasc_full.name[:-3]}_near_neighbor_ids.fits.gz" t = Table([list(near_ids)], names=["near_id"]) print(f"Writing {len(t)} near-neighbor IDs to {outfile}") t.write(str(outfile), format="fits", overwrite=True) diff --git a/dev/profile_memory_psrecord.py b/dev/profile_memory_psrecord.py index 040f206a..82217d08 100644 --- a/dev/profile_memory_psrecord.py +++ b/dev/profile_memory_psrecord.py @@ -2,8 +2,10 @@ print("importing time") import time + t0 = time.time() + def wait_until_time(t1): t_now = time.time() time.sleep(t1 - (t_now - t0)) diff --git a/scripts/make_fits.py b/scripts/make_fits.py index 57ab7772..a4057238 100644 --- a/scripts/make_fits.py +++ b/scripts/make_fits.py @@ -25,12 +25,14 @@ ' ', ' ', 'File written on Sept 5 2018', - ' ') + ' ', +) def get_options(): parser = argparse.ArgumentParser( - description="Make new AGASC fits files from old AGASC and new h5 source") + description="Make new AGASC fits files from old AGASC and new h5 source" + ) parser.add_argument("--h5", default='/proj/sot/ska/data/agasc/agasc1p7.h5') parser.add_argument("--olddir", default="/data/agasc1p6/agasc") parser.add_argument("--out", default="./testagasc") @@ -93,7 +95,7 @@ def main(h5file, srcdir, outdir): newfile = newdir.joinpath(f.name) update_file(f, newfile, tbl) + if __name__ == '__main__': args = get_options() main(args.h5, args.olddir, args.out) - diff --git a/scripts/make_links.py b/scripts/make_links.py index 587788ce..36722a96 100644 --- a/scripts/make_links.py +++ b/scripts/make_links.py @@ -12,4 +12,3 @@ newname = ffile.name.upper() os.symlink((".." / ffile), newname) os.chdir("..") - diff --git a/setup.py b/setup.py index da0c324b..ab60e64a 100644 --- a/setup.py +++ b/setup.py @@ -19,28 +19,38 @@ } -data_files = [( - os.path.join('share', 'agasc'), - ['task_schedules/task_schedule_supplement_dispositions.cfg', - 'task_schedules/task_schedule_update_supplement_rc.cfg'] -)] +data_files = [ + ( + os.path.join('share', 'agasc'), + [ + 'task_schedules/task_schedule_supplement_dispositions.cfg', + 'task_schedules/task_schedule_update_supplement_rc.cfg', + ], + ) +] -setup(name='agasc', - use_scm_version=True, - setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], - description='AGASC catalog access', - author='Jean Connelly, Tom Aldcroft', - author_email='taldcroft@cfa.harvard.edu', - url='http://cxc.harvard.edu/mta/ASPECT/tool_doc/agasc', - packages=['agasc', 'agasc.supplement', 'agasc.supplement.magnitudes', - 'agasc.tests', 'agasc.scripts'], - package_data={ - 'agasc.supplement.magnitudes': ['templates/*'], - 'agasc.tests': ['data/*'], - }, - data_files=data_files, - tests_require=['pytest'], - cmdclass=cmdclass, - entry_points=entry_points, - ) +setup( + name='agasc', + use_scm_version=True, + setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], + description='AGASC catalog access', + author='Jean Connelly, Tom Aldcroft', + author_email='taldcroft@cfa.harvard.edu', + url='http://cxc.harvard.edu/mta/ASPECT/tool_doc/agasc', + packages=[ + 'agasc', + 'agasc.supplement', + 'agasc.supplement.magnitudes', + 'agasc.tests', + 'agasc.scripts', + ], + package_data={ + 'agasc.supplement.magnitudes': ['templates/*'], + 'agasc.tests': ['data/*'], + }, + data_files=data_files, + tests_require=['pytest'], + cmdclass=cmdclass, + entry_points=entry_points, +) From 70e495b25722fc1e00960a8292437c608ceab72c Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:40:09 -0400 Subject: [PATCH 05/10] black . --- agasc/__init__.py | 4 +- agasc/agasc.py | 116 +- agasc/paths.py | 10 +- agasc/scripts/mag_estimate_report.py | 74 +- agasc/scripts/supplement_diff.py | 46 +- agasc/scripts/supplement_tasks.py | 34 +- agasc/scripts/update_mag_supplement.py | 112 +- agasc/scripts/update_supplement.py | 168 +-- agasc/supplement/magnitudes/mag_estimate.py | 1325 ++++++++--------- .../magnitudes/mag_estimate_report.py | 704 ++++----- .../magnitudes/star_obs_catalogs.py | 30 +- .../magnitudes/update_mag_supplement.py | 322 ++-- agasc/supplement/utils.py | 134 +- agasc/tests/test_agasc_1.py | 44 +- agasc/tests/test_agasc_2.py | 290 ++-- agasc/tests/test_obs_status.py | 1310 ++++++++-------- scripts/make_fits.py | 70 +- scripts/make_links.py | 2 +- setup.py | 46 +- 19 files changed, 2420 insertions(+), 2421 deletions(-) diff --git a/agasc/__init__.py b/agasc/__init__.py index 93221cef..5160c538 100644 --- a/agasc/__init__.py +++ b/agasc/__init__.py @@ -9,9 +9,9 @@ def test(*args, **kwargs): - ''' + """ Run py.test unit tests. - ''' + """ import testr return testr.test(*args, **kwargs) diff --git a/agasc/agasc.py b/agasc/agasc.py index 85ede00a..9573366a 100644 --- a/agasc/agasc.py +++ b/agasc/agasc.py @@ -18,20 +18,20 @@ from .supplement.utils import get_supplement_table __all__ = [ - 'sphere_dist', - 'get_agasc_cone', - 'get_star', - 'get_stars', - 'read_h5_table', - 'get_agasc_filename', - 'MAG_CATID_SUPPLEMENT', - 'BAD_CLASS_SUPPLEMENT', - 'set_supplement_enabled', - 'SUPPLEMENT_ENABLED_ENV', + "sphere_dist", + "get_agasc_cone", + "get_star", + "get_stars", + "read_h5_table", + "get_agasc_filename", + "MAG_CATID_SUPPLEMENT", + "BAD_CLASS_SUPPLEMENT", + "set_supplement_enabled", + "SUPPLEMENT_ENABLED_ENV", ] -SUPPLEMENT_ENABLED_ENV = 'AGASC_SUPPLEMENT_ENABLED' -SUPPLEMENT_ENABLED_DEFAULT = 'True' +SUPPLEMENT_ENABLED_ENV = "AGASC_SUPPLEMENT_ENABLED" +SUPPLEMENT_ENABLED_DEFAULT = "True" MAG_CATID_SUPPLEMENT = 100 BAD_CLASS_SUPPLEMENT = 100 @@ -107,7 +107,7 @@ def test_get_aca_catalog(): Whether to use the AGASC supplement in the context / decorator """ if not isinstance(value, bool): - raise TypeError('value must be bool (True|False)') + raise TypeError("value must be bool (True|False)") orig = os.environ.get(SUPPLEMENT_ENABLED_ENV) os.environ[SUPPLEMENT_ENABLED_ENV] = str(value) @@ -137,21 +137,21 @@ def agasc_file(self): @property def ra(self): - if not hasattr(self, '_ra'): + if not hasattr(self, "_ra"): self._ra, self._dec = self.read_ra_dec() return self._ra @property def dec(self): - if not hasattr(self, '_dec'): + if not hasattr(self, "_dec"): self._ra, self._dec = self.read_ra_dec() return self._dec def read_ra_dec(self): # Read the RA and DEC values from the agasc with tables.open_file(self.agasc_file) as h5: - ras = h5.root.data.read(field='RA') - decs = h5.root.data.read(field='DEC') + ras = h5.root.data.read(field="RA") + decs = h5.root.data.read(field="DEC") return ras, decs @@ -368,11 +368,11 @@ def sphere_dist(ra1, dec1, ra2, dec2): dec2 = np.radians(dec2).astype(np.float64) numerator = numexpr.evaluate( - 'sin((dec2 - dec1) / 2) ** 2 + ' # noqa - 'cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2' + "sin((dec2 - dec1) / 2) ** 2 + " # noqa + "cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2" ) - dists = numexpr.evaluate('2 * arctan2(numerator ** 0.5, (1 - numerator) ** 0.5)') + dists = numexpr.evaluate("2 * arctan2(numerator ** 0.5, (1 - numerator) ** 0.5)") return np.degrees(dists) @@ -392,8 +392,8 @@ def update_color1_column(stars): This updates ``stars`` in place. """ # Select red stars that have a reliable mag in AGASC 1.7 and later. - color15 = np.isclose(stars['COLOR1'], 1.5) & (stars['RSV3'] > 0) - new_color1 = stars['COLOR2'][color15] * 0.850 + color15 = np.isclose(stars["COLOR1"], 1.5) & (stars["RSV3"] > 0) + new_color1 = stars["COLOR2"][color15] * 0.850 if len(new_color1) > 0: # Ensure no new COLOR1 are within 0.001 of 1.5, so downstream tests of @@ -402,7 +402,7 @@ def update_color1_column(stars): new_color1[fix15] = 1.499 # Insignificantly different from 1.50 # For stars with a reliable mag, now COLOR1 is really the B-V color. - stars['COLOR1'][color15] = new_color1 + stars["COLOR1"][color15] = new_color1 def add_pmcorr_columns(stars, date): @@ -426,26 +426,26 @@ def add_pmcorr_columns(stars, date): # Compute delta year. stars['EPOCH'] is Column, float32. Need to coerce to # ndarray float64 for consistent results between scalar and array cases. - dyear = dates.frac_year - stars['EPOCH'].view(np.ndarray).astype(np.float64) + dyear = dates.frac_year - stars["EPOCH"].view(np.ndarray).astype(np.float64) pm_to_degrees = dyear / (3600.0 * 1000.0) dec_pmcorr = np.where( - stars['PM_DEC'] != -9999, - stars['DEC'] + stars['PM_DEC'] * pm_to_degrees, - stars['DEC'], + stars["PM_DEC"] != -9999, + stars["DEC"] + stars["PM_DEC"] * pm_to_degrees, + stars["DEC"], ) - ra_scale = np.cos(np.radians(stars['DEC'])) + ra_scale = np.cos(np.radians(stars["DEC"])) ra_pmcorr = np.where( - stars['PM_RA'] != -9999, - stars['RA'] + stars['PM_RA'] * pm_to_degrees / ra_scale, - stars['RA'], + stars["PM_RA"] != -9999, + stars["RA"] + stars["PM_RA"] * pm_to_degrees / ra_scale, + stars["RA"], ) # Add the proper-motion corrected columns to table using astropy.table.Table stars.add_columns( [ - Column(data=ra_pmcorr, name='RA_PMCORR'), - Column(data=dec_pmcorr, name='DEC_PMCORR'), + Column(data=ra_pmcorr, name="RA_PMCORR"), + Column(data=dec_pmcorr, name="DEC_PMCORR"), ] ) @@ -507,7 +507,7 @@ def get_agasc_cone( # Final filtering using proper-motion corrected positions if pm_filter: - dists = sphere_dist(ra, dec, stars['RA_PMCORR'], stars['DEC_PMCORR']) + dists = sphere_dist(ra, dec, stars["RA_PMCORR"], stars["DEC_PMCORR"]) ok = dists <= radius stars = stars[ok] @@ -599,7 +599,7 @@ def get_star(id, agasc_file=None, date=None, fix_color1=True, use_supplement=Non with tables.open_file(agasc_file) as h5: tbl = h5.root.data - id_rows = tbl.read_where('(AGASC_ID == {})'.format(id)) + id_rows = tbl.read_where("(AGASC_ID == {})".format(id)) if len(id_rows) > 1: raise InconsistentCatalogError( @@ -624,15 +624,15 @@ def _get_rows_read_where(ids_1d, dates_1d, agasc_file): with tables.open_file(agasc_file) as h5: tbl = h5.root.data for id, date in zip(ids_1d, dates_1d): - id_rows = tbl.read_where('(AGASC_ID == {})'.format(id)) + id_rows = tbl.read_where("(AGASC_ID == {})".format(id)) if len(id_rows) > 1: raise InconsistentCatalogError( - f'More than one entry found for {id} in AGASC' + f"More than one entry found for {id} in AGASC" ) if id_rows is None or len(id_rows) == 0: - raise IdNotFound(f'No entry found for {id} in AGASC') + raise IdNotFound(f"No entry found for {id} in AGASC") rows.append(id_rows[0]) return rows @@ -642,12 +642,12 @@ def _get_rows_read_entire(ids_1d, dates_1d, agasc_file): with tables.open_file(agasc_file) as h5: tbl = h5.root.data[:] - agasc_idx = {agasc_id: idx for idx, agasc_id in enumerate(tbl['AGASC_ID'])} + agasc_idx = {agasc_id: idx for idx, agasc_id in enumerate(tbl["AGASC_ID"])} rows = [] for agasc_id, date in zip(ids_1d, dates_1d): if agasc_id not in agasc_idx: - raise IdNotFound(f'No entry found for {agasc_id} in AGASC') + raise IdNotFound(f"No entry found for {agasc_id} in AGASC") rows.append(tbl[agasc_idx[agasc_id]]) return rows @@ -721,10 +721,10 @@ def get_stars( if len(ids_1d) < method_threshold: rows = _get_rows_read_where(ids_1d, dates_1d, agasc_file) - method = 'tables_read_where' + method = "tables_read_where" else: rows = _get_rows_read_entire(ids_1d, dates_1d, agasc_file) - method = 'read_entire_agasc' + method = "read_entire_agasc" t = Table(np.vstack(rows).flatten()) @@ -734,7 +734,7 @@ def get_stars( add_pmcorr_columns(t, dates_in if dates_is_scalar else dates) if fix_color1: update_color1_column(t) - t['DATE'] = dates + t["DATE"] = dates update_from_supplement(t, use_supplement) @@ -780,12 +780,12 @@ def update_from_supplement(stars, use_supplement=None): supplement_enabled_env = os.environ.get( SUPPLEMENT_ENABLED_ENV, SUPPLEMENT_ENABLED_DEFAULT ) - if supplement_enabled_env not in ('True', 'False'): + if supplement_enabled_env not in ("True", "False"): raise ValueError( f'{SUPPLEMENT_ENABLED_ENV} env var must be either "True" or "False" ' - f'got {supplement_enabled_env}' + f"got {supplement_enabled_env}" ) - supplement_enabled = supplement_enabled_env == 'True' + supplement_enabled = supplement_enabled_env == "True" else: supplement_enabled = use_supplement @@ -801,28 +801,28 @@ def set_star(star, name, value): # Get estimate mags and errs from supplement as a dict of dict # agasc_id : {mag_aca: .., mag_aca_err: ..}. - supplement_mags = get_supplement_table('mags', agasc_dir=default_agasc_dir()) + supplement_mags = get_supplement_table("mags", agasc_dir=default_agasc_dir()) supplement_mags_index = supplement_mags.meta["index"] # Get bad stars as {agasc_id: {source: ..}} - bad_stars = get_supplement_table('bad', agasc_dir=default_agasc_dir()) + bad_stars = get_supplement_table("bad", agasc_dir=default_agasc_dir()) bad_stars_index = bad_stars.meta["index"] for star in stars: - agasc_id = int(star['AGASC_ID']) + agasc_id = int(star["AGASC_ID"]) if agasc_id in supplement_mags_index: idx = supplement_mags_index[agasc_id] - mag_est = supplement_mags['mag_aca'][idx] - mag_est_err = supplement_mags['mag_aca_err'][idx] + mag_est = supplement_mags["mag_aca"][idx] + mag_est_err = supplement_mags["mag_aca_err"][idx] - set_star(star, 'MAG_ACA', mag_est) + set_star(star, "MAG_ACA", mag_est) # Mag err is stored as int16 in units of 0.01 mag. Use same convention here. - set_star(star, 'MAG_ACA_ERR', round(mag_est_err * 100)) - set_star(star, 'MAG_CATID', MAG_CATID_SUPPLEMENT) - if 'COLOR1' in stars.colnames: - color1 = star['COLOR1'] + set_star(star, "MAG_ACA_ERR", round(mag_est_err * 100)) + set_star(star, "MAG_CATID", MAG_CATID_SUPPLEMENT) + if "COLOR1" in stars.colnames: + color1 = star["COLOR1"] if np.isclose(color1, 0.7) or np.isclose(color1, 1.5): - star['COLOR1'] = color1 - 0.01 + star["COLOR1"] = color1 - 0.01 if agasc_id in bad_stars_index: - set_star(star, 'CLASS', BAD_CLASS_SUPPLEMENT) + set_star(star, "CLASS", BAD_CLASS_SUPPLEMENT) diff --git a/agasc/paths.py b/agasc/paths.py index 1e377201..c2a4744e 100644 --- a/agasc/paths.py +++ b/agasc/paths.py @@ -2,10 +2,10 @@ import os from pathlib import Path -SUPPLEMENT_FILENAME = 'agasc_supplement.h5' +SUPPLEMENT_FILENAME = "agasc_supplement.h5" -__all__ = ['default_agasc_dir', 'default_agasc_file', 'SUPPLEMENT_FILENAME'] +__all__ = ["default_agasc_dir", "default_agasc_file", "SUPPLEMENT_FILENAME"] def default_agasc_dir(): @@ -16,10 +16,10 @@ def default_agasc_dir(): :returns: Path """ - if 'AGASC_DIR' in os.environ: - out = Path(os.environ['AGASC_DIR']) + if "AGASC_DIR" in os.environ: + out = Path(os.environ["AGASC_DIR"]) else: - out = Path(os.environ['SKA'], 'data', 'agasc') + out = Path(os.environ["SKA"], "data", "agasc") return out diff --git a/agasc/scripts/mag_estimate_report.py b/agasc/scripts/mag_estimate_report.py index e59d340e..9bd180fa 100755 --- a/agasc/scripts/mag_estimate_report.py +++ b/agasc/scripts/mag_estimate_report.py @@ -17,57 +17,57 @@ def get_parser(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - '--start', + "--start", help=( - 'Time to start processing new observations.' - ' CxoTime-compatible time stamp.' - ' Default: now - 90 days' + "Time to start processing new observations." + " CxoTime-compatible time stamp." + " Default: now - 90 days" ), ) parser.add_argument( - '--stop', + "--stop", help=( - 'Time to stop processing new observations.' - ' CxoTime-compatible time stamp.' - ' Default: now' + "Time to stop processing new observations." + " CxoTime-compatible time stamp." + " Default: now" ), ) parser.add_argument( - '--input-dir', - default='$SKA/data/agasc', - help='Directory containing mag-stats files. Default: $SKA/data/agasc', + "--input-dir", + default="$SKA/data/agasc", + help="Directory containing mag-stats files. Default: $SKA/data/agasc", ) parser.add_argument( - '--output-dir', - default='supplement_reports/suspect', - help='Output directory. Default: supplement_reports/suspect', + "--output-dir", + default="supplement_reports/suspect", + help="Output directory. Default: supplement_reports/suspect", ) parser.add_argument( - '--obs-stats', - default='mag_stats_obsid.fits', + "--obs-stats", + default="mag_stats_obsid.fits", help=( - 'FITS file with mag-stats for all observations.' - ' Default: mag_stats_obsid.fits' + "FITS file with mag-stats for all observations." + " Default: mag_stats_obsid.fits" ), ) parser.add_argument( - '--agasc-stats', - default='mag_stats_agasc.fits', + "--agasc-stats", + default="mag_stats_agasc.fits", help=( - 'FITS file with mag-stats for all observed AGASC stars.' - ' Default: mag_stats_agasc.fits' + "FITS file with mag-stats for all observed AGASC stars." + " Default: mag_stats_agasc.fits" ), ) parser.add_argument( - '--weekly-report', + "--weekly-report", help="Add links to navigate weekly reports.", - action='store_true', + action="store_true", default=False, ) parser.add_argument( - '--all-stars', + "--all-stars", help="Include all stars in the report, not just suspect.", - action='store_true', + action="store_true", default=False, ) return parser @@ -76,7 +76,7 @@ def get_parser(): def main(): import kadi.commands - kadi.commands.conf.commands_version = '1' + kadi.commands.conf.commands_version = "1" args = get_parser().parse_args() @@ -96,23 +96,23 @@ def main(): else: args.start = CxoTime(args.start) - t = obs_stats['mp_starcat_time'] + t = obs_stats["mp_starcat_time"] ok = (t < args.stop) & (t > args.start) if not args.all_stars: - ok &= ~obs_stats['obs_ok'] - stars = np.unique(obs_stats[ok]['agasc_id']) - sections = [{'id': 'stars', 'title': 'Stars', 'stars': stars}] + ok &= ~obs_stats["obs_ok"] + stars = np.unique(obs_stats[ok]["agasc_id"]) + sections = [{"id": "stars", "title": "Stars", "stars": stars}] - agasc_stats = agasc_stats[np.in1d(agasc_stats['agasc_id'], stars)] + agasc_stats = agasc_stats[np.in1d(agasc_stats["agasc_id"], stars)] if args.weekly_report: t = CxoTime(args.stop) directory = args.output_dir / t.date[:8] week = time.TimeDelta(7 * u.day) nav_links = { - 'previous': f'../{(t - week).date[:8]}', - 'up': '..', - 'next': f'../{(t + week).date[:8]}', + "previous": f"../{(t - week).date[:8]}", + "up": "..", + "next": f"../{(t + week).date[:8]}", } else: directory = args.output_dir @@ -125,10 +125,10 @@ def main(): sections=sections, tstart=args.start, tstop=args.stop, - filename='index.html', + filename="index.html", nav_links=nav_links, ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/scripts/supplement_diff.py b/agasc/scripts/supplement_diff.py index b3fc8db0..9d822459 100755 --- a/agasc/scripts/supplement_diff.py +++ b/agasc/scripts/supplement_diff.py @@ -21,12 +21,12 @@ def read_file(filename, exclude=[]): formats = { - 'agasc_versions': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']}, - 'last_updated': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']}, - 'obs': {'comments': '{:>80s}', 'agasc_id': '{:10d}'}, + "agasc_versions": {k: "{:>21s}" for k in ["mags", "obs", "bad", "supplement"]}, + "last_updated": {k: "{:>21s}" for k in ["mags", "obs", "bad", "supplement"]}, + "obs": {"comments": "{:>80s}", "agasc_id": "{:10d}"}, } - node_names = ['agasc_versions', 'last_updated', 'obs', 'bad'] + node_names = ["agasc_versions", "last_updated", "obs", "bad"] with tables.open_file(filename) as h5: for node in h5.root: if node.name not in node_names: @@ -35,32 +35,32 @@ def read_file(filename, exclude=[]): for name in node_names: if name in exclude: continue - node = h5.get_node(f'/{name}') + node = h5.get_node(f"/{name}") t = table.Table(node[:]) t.convert_bytestring_to_unicode s = StringIO() - ascii.write(t, s, format='fixed_width', formats=formats.get(name, {})) + ascii.write(t, s, format="fixed_width", formats=formats.get(name, {})) s.seek(0) lines = s.readlines() dashes = "-" * (len(lines[0]) - 1) + "\n" - all_lines += [dashes, f'| {node.name}\n', dashes] + lines + [dashes] + all_lines += [dashes, f"| {node.name}\n", dashes] + lines + [dashes] return all_lines def diff_files(fromfile, tofile, exclude_mags=False): - exclude = ['mags'] if exclude_mags else [] + exclude = ["mags"] if exclude_mags else [] fromlines = read_file(fromfile, exclude) tolines = read_file(tofile, exclude) diff = difflib.unified_diff(fromlines, tolines, str(fromfile), str(tofile)) - diff = ''.join(diff) + diff = "".join(diff) return diff def diff_to_html(diff): - date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") lexer = pygments.lexers.DiffLexer() formatter = pygments.formatters.HtmlFormatter( - full=True, linenos='table', title=f'AGASC supplement diff - {date}' + full=True, linenos="table", title=f"AGASC supplement diff - {date}" ) return pygments.highlight(diff, lexer, formatter) @@ -68,18 +68,18 @@ def diff_to_html(diff): def get_parser(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - '--from', dest='fromfile', type=Path, help='The original supplement file.' + "--from", dest="fromfile", type=Path, help="The original supplement file." ) parser.add_argument( - '--to', dest='tofile', type=Path, help='The modified supplement file.' + "--to", dest="tofile", type=Path, help="The modified supplement file." ) parser.add_argument( - '-o', help='Output HTML file', type=Path, default='agasc_supplement_diff.html' + "-o", help="Output HTML file", type=Path, default="agasc_supplement_diff.html" ) parser.add_argument( - '--exclude-mags', + "--exclude-mags", default=False, - action='store_true', + action="store_true", help='Exclude changes in the "mags" table. There can be many of these.', ) return parser @@ -87,23 +87,23 @@ def get_parser(): def main(): args = get_parser().parse_args() - if args.fromfile is None and 'SKA' in os.environ: + if args.fromfile is None and "SKA" in os.environ: args.fromfile = ( - Path(os.environ['SKA']) / 'data' / 'agasc' / 'agasc_supplement.h5' + Path(os.environ["SKA"]) / "data" / "agasc" / "agasc_supplement.h5" ) - if args.tofile is None and 'SKA' in os.environ: + if args.tofile is None and "SKA" in os.environ: args.tofile = ( - Path(os.environ['SKA']) / 'data' / 'agasc' / 'rc' / 'agasc_supplement.h5' + Path(os.environ["SKA"]) / "data" / "agasc" / "rc" / "agasc_supplement.h5" ) assert args.tofile, 'Option "--to" was not given and SKA is not defined' assert args.fromfile, 'Option "--from" was not given and SKA is not defined' - assert args.fromfile.exists(), f'File {args.fromfile} does not exist' - assert args.tofile.exists(), f'File {args.tofile} does not exist' + assert args.fromfile.exists(), f"File {args.fromfile} does not exist" + assert args.tofile.exists(), f"File {args.tofile} does not exist" if not args.o.parent.exists(): args.o.parent.mkdir() - with open(args.o, 'w') as fh: + with open(args.o, "w") as fh: diff = diff_to_html( diff_files(args.fromfile, args.tofile, exclude_mags=args.exclude_mags) ) diff --git a/agasc/scripts/supplement_tasks.py b/agasc/scripts/supplement_tasks.py index 17694c73..c7fc9dc8 100644 --- a/agasc/scripts/supplement_tasks.py +++ b/agasc/scripts/supplement_tasks.py @@ -21,7 +21,7 @@ from cxotime import CxoTime -AGASC_DATA = Path(os.environ['SKA']) / 'data' / 'agasc' +AGASC_DATA = Path(os.environ["SKA"]) / "data" / "agasc" def email_promotion_report( @@ -44,14 +44,14 @@ def update_rc(): """ Update the supplement in $SKA/data/agasc/rc """ - filenames = list((AGASC_DATA / 'rc' / 'promote').glob('*')) + filenames = list((AGASC_DATA / "rc" / "promote").glob("*")) if filenames: for file in filenames: file.rename(AGASC_DATA / file.name) - email_promotion_report(filenames, destdir=AGASC_DATA, to='aca@cfa.harvard.edu') + email_promotion_report(filenames, destdir=AGASC_DATA, to="aca@cfa.harvard.edu") subprocess.run( - ['task_schedule3.pl', '-config', 'agasc/task_schedule_update_supplement_rc.cfg'] + ["task_schedule3.pl", "-config", "agasc/task_schedule_update_supplement_rc.cfg"] ) @@ -63,9 +63,9 @@ def disposition(): """ subprocess.run( [ - 'task_schedule3.pl', - '-config', - 'agasc/task_schedule_supplement_dispositions.cfg', + "task_schedule3.pl", + "-config", + "agasc/task_schedule_supplement_dispositions.cfg", ] ) @@ -77,21 +77,21 @@ def stage_promotion(): It just copies the files into $SKA/data/agasc/rc/promote. The promotion task_schedule will move them to $SKA/data/agasc. """ - promote_dir = AGASC_DATA / 'rc' / 'promote' - rc_dir = AGASC_DATA / 'rc' + promote_dir = AGASC_DATA / "rc" / "promote" + rc_dir = AGASC_DATA / "rc" promote_dir.mkdir(exist_ok=True) for filename in [ - 'agasc_supplement.h5', - 'mag_stats_agasc.fits', - 'mag_stats_obsid.fits', + "agasc_supplement.h5", + "mag_stats_agasc.fits", + "mag_stats_obsid.fits", ]: shutil.copy(rc_dir / filename, promote_dir / filename) TASKS = { - 'update-rc': update_rc, - 'disposition': disposition, - 'schedule-promotion': stage_promotion, + "update-rc": update_rc, + "disposition": disposition, + "schedule-promotion": stage_promotion, } @@ -99,7 +99,7 @@ def get_parser(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('task', choices=TASKS) + parser.add_argument("task", choices=TASKS) return parser @@ -108,5 +108,5 @@ def main(): TASKS[args.task]() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/scripts/update_mag_supplement.py b/agasc/scripts/update_mag_supplement.py index 5ca46ecd..1393e5b4 100755 --- a/agasc/scripts/update_mag_supplement.py +++ b/agasc/scripts/update_mag_supplement.py @@ -23,84 +23,84 @@ def get_parser(): description=__doc__, parents=[update_supplement.get_obs_status_parser()] ) parser.add_argument( - '--start', + "--start", help=( - 'Include only stars observed after this time.' - ' CxoTime-compatible time stamp.' - ' Default: now - 30 days.' + "Include only stars observed after this time." + " CxoTime-compatible time stamp." + " Default: now - 30 days." ), ) parser.add_argument( - '--stop', + "--stop", help=( - 'Include only stars observed before this time.' - ' CxoTime-compatible time stamp.' - ' Default: now.' + "Include only stars observed before this time." + " CxoTime-compatible time stamp." + " Default: now." ), ) parser.add_argument( - '--whole-history', - help='Include all star observations and ignore --start/stop.', - action='store_true', + "--whole-history", + help="Include all star observations and ignore --start/stop.", + action="store_true", default=False, ) parser.add_argument( - '--agasc-id-file', + "--agasc-id-file", help=( - 'Include only observations of stars whose AGASC IDs are specified ' - 'in this file, one per line.' + "Include only observations of stars whose AGASC IDs are specified " + "in this file, one per line." ), ) parser.add_argument( - '--output-dir', + "--output-dir", help=( - 'Directory where agasc_supplement.h5 is located.' - 'Other output is placed here as well. Default: .' + "Directory where agasc_supplement.h5 is located." + "Other output is placed here as well. Default: ." ), - default='.', + default=".", ) parser.add_argument( - '--include-bad', + "--include-bad", help='Do not exclude "bad" stars from magnitude estimates. Default: False', - action='store_true', + action="store_true", default=False, ) - report = parser.add_argument_group('Reporting') + report = parser.add_argument_group("Reporting") report.add_argument( - '--report', - help='Generate HTML report for the period covered. Default: False', - action='store_true', + "--report", + help="Generate HTML report for the period covered. Default: False", + action="store_true", default=False, ) report.add_argument( - '--reports-dir', + "--reports-dir", help=( - 'Directory where to place reports.' - ' Default: /supplement_reports/weekly.' + "Directory where to place reports." + " Default: /supplement_reports/weekly." ), ) - other = parser.add_argument_group('Other') + other = parser.add_argument_group("Other") other.add_argument( - '--multi-process', + "--multi-process", help="Use multi-processing to accelerate run.", - action='store_true', + action="store_true", default=False, ) other.add_argument( - '--log-level', default='info', choices=['debug', 'info', 'warning', 'error'] + "--log-level", default="info", choices=["debug", "info", "warning", "error"] ) other.add_argument( - '--no-progress', - dest='no_progress', - help='Do not show a progress bar', - action='store_true', + "--no-progress", + dest="no_progress", + help="Do not show a progress bar", + action="store_true", ) # this has no default, it will be None. other.add_argument( - '--args-file', + "--args-file", help=( - 'YAML file with arguments to ' - 'agasc.supplement.magnitudes.update_mag_supplement.do' + "YAML file with arguments to " + "agasc.supplement.magnitudes.update_mag_supplement.do" ), ) other.add_argument( @@ -114,42 +114,42 @@ def get_parser(): def main(): import kadi.commands - kadi.commands.conf.commands_version = '1' + kadi.commands.conf.commands_version = "1" - logger = logging.getLogger('agasc.supplement') + logger = logging.getLogger("agasc.supplement") the_parser = get_parser() args = the_parser.parse_args() if args.args_file: args.args_file = Path(args.args_file) if not args.args_file.exists(): - logger.error(f'File does not exist: {args.args_file}') + logger.error(f"File does not exist: {args.args_file}") the_parser.exit(1) with open(args.args_file) as fh: file_args = yaml.load(fh, Loader=yaml.SafeLoader) the_parser.set_defaults(**file_args) args = the_parser.parse_args() - status_to_int = {'true': 1, 'false': 0, 'ok': 1, 'good': 1, 'bad': 0} + status_to_int = {"true": 1, "false": 0, "ok": 1, "good": 1, "bad": 0} if args.status and args.status.lower() in status_to_int: args.status = status_to_int[args.status.lower()] args.output_dir = Path(os.path.expandvars(args.output_dir)) if args.reports_dir is None: - args.reports_dir = args.output_dir / 'supplement_reports' / 'weekly' + args.reports_dir = args.output_dir / "supplement_reports" / "weekly" else: args.reports_dir = Path(os.path.expandvars(args.reports_dir)) if args.whole_history: if args.start or args.stop: logger.error( - '--whole-history argument is incompatible with --start/--stop arguments' + "--whole-history argument is incompatible with --start/--stop arguments" ) the_parser.exit(1) args.start = None args.stop = None pyyaks.logger.get_logger( - name='agasc.supplement', + name="agasc.supplement", level=args.log_level.upper(), format="%(asctime)s %(message)s", ) @@ -158,8 +158,8 @@ def main(): not (args.obsid or args.mp_starcat_time) and args.status ): logger.error( - 'To override OBS status, both --obs/mp-starcat-time and --status options' - ' are needed.' + "To override OBS status, both --obs/mp-starcat-time and --status options" + " are needed." ) the_parser.exit(1) @@ -168,7 +168,7 @@ def main(): # set the list of AGASC IDs from file if specified. If not, it will include all. agasc_ids = [] if args.agasc_id_file: - with open(args.agasc_id_file, 'r') as f: + with open(args.agasc_id_file, "r") as f: agasc_ids = [int(line.strip()) for line in f.readlines()] # update 'bad' and 'obs' tables in supplement @@ -177,9 +177,9 @@ def main(): # set start/stop times if args.whole_history: if args.start or args.stop: - raise ValueError('incompatible arguments: whole_history and start/stop') - args.start = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).min().date - args.stop = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).max().date + raise ValueError("incompatible arguments: whole_history and start/stop") + args.start = CxoTime(star_obs_catalogs.STARS_OBS["mp_starcat_time"]).min().date + args.stop = CxoTime(star_obs_catalogs.STARS_OBS["mp_starcat_time"]).max().date else: args.stop = CxoTime(args.stop).date if args.stop else CxoTime.now().date args.start = ( @@ -196,10 +196,10 @@ def main(): report_date += ((7 - report_date.datetime.weekday()) % 7) * u.day report_date = CxoTime(report_date.date[:8]) - args_log_file = args.output_dir / 'call_args.yml' + args_log_file = args.output_dir / "call_args.yml" if not args.output_dir.exists(): args.output_dir.mkdir(parents=True) - with open(args_log_file, 'w') as fh: + with open(args_log_file, "w") as fh: # there must be a better way to do this... yaml_args = { k: str(v) if issubclass(type(v), Path) else v for k, v in vars(args).items() @@ -219,11 +219,11 @@ def main(): dry_run=args.dry_run, no_progress=args.no_progress, ) - if args.report and (args.reports_dir / f'{report_date.date[:8]}').exists(): + if args.report and (args.reports_dir / f"{report_date.date[:8]}").exists(): args_log_file.replace( - args.reports_dir / f'{report_date.date[:8]}' / args_log_file.name + args.reports_dir / f"{report_date.date[:8]}" / args_log_file.name ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/scripts/update_supplement.py b/agasc/scripts/update_supplement.py index aa448bf9..308b6ff8 100755 --- a/agasc/scripts/update_supplement.py +++ b/agasc/scripts/update_supplement.py @@ -21,7 +21,7 @@ from agasc.supplement.utils import update_mags_table, update_obs_table, add_bad_star -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") def _parse_obs_status_file(filename): @@ -35,76 +35,76 @@ def _parse_obs_status_file(filename): """ with open(filename) as fh: status = yaml.load(fh, Loader=yaml.SafeLoader) - if 'obs' not in status: - status['obs'] = [] - if 'bad' not in status: - status['bad'] = [] - if 'mags' not in status: - status['mags'] = [] + if "obs" not in status: + status["obs"] = [] + if "bad" not in status: + status["bad"] = [] + if "mags" not in status: + status["mags"] = [] - if hasattr(status['bad'], 'items'): - status['bad'] = list(status['bad'].items()) + if hasattr(status["bad"], "items"): + status["bad"] = list(status["bad"].items()) return status def _sanitize_args(status): - for star in status['mags']: - if 'last_obs_time' in star: - star['last_obs_time'] = CxoTime(star['last_obs_time']).cxcsec + for star in status["mags"]: + if "last_obs_time" in star: + star["last_obs_time"] = CxoTime(star["last_obs_time"]).cxcsec else: - obs = cat.STARS_OBS[cat.STARS_OBS['agasc_id'] == star['agasc_id']] + obs = cat.STARS_OBS[cat.STARS_OBS["agasc_id"] == star["agasc_id"]] if len(obs) == 0: raise Exception( f"Can not guess last_obs_time for agasc_id={star['agasc_id']}" ) - star['last_obs_time'] = CxoTime(obs['mp_starcat_time']).max().cxcsec + star["last_obs_time"] = CxoTime(obs["mp_starcat_time"]).max().cxcsec - for value in status['obs']: + for value in status["obs"]: if cat.STARS_OBS is None: - raise RuntimeError('Observation catalog is not initialized') - if 'mp_starcat_time' in value: - key = 'mp_starcat_time' + raise RuntimeError("Observation catalog is not initialized") + if "mp_starcat_time" in value: + key = "mp_starcat_time" rows = cat.STARS_OBS[ - cat.STARS_OBS['mp_starcat_time'] == value['mp_starcat_time'] + cat.STARS_OBS["mp_starcat_time"] == value["mp_starcat_time"] ] - elif 'obsid' in value: - key = 'obsid' - rows = cat.STARS_OBS[cat.STARS_OBS['obsid'] == value['obsid']] + elif "obsid" in value: + key = "obsid" + rows = cat.STARS_OBS[cat.STARS_OBS["obsid"] == value["obsid"]] else: - raise Exception('Need to specify mp_starcat_time or OBSID') + raise Exception("Need to specify mp_starcat_time or OBSID") if len(rows) == 0: raise Exception( - f'Observation catalog has no observation with {key}={value[key]}' + f"Observation catalog has no observation with {key}={value[key]}" ) - if 'agasc_id' not in value: - value['agasc_id'] = list(sorted(rows['agasc_id'])) + if "agasc_id" not in value: + value["agasc_id"] = list(sorted(rows["agasc_id"])) else: - value['agasc_id'] = list(np.atleast_1d(value['agasc_id'])) - if 'comments' not in value: - value['comments'] = '' - if 'mp_starcat_time' not in value: - value['mp_starcat_time'] = rows['mp_starcat_time'][0] - if 'obsid' not in value: - value['obsid'] = rows['obsid'][0] + value["agasc_id"] = list(np.atleast_1d(value["agasc_id"])) + if "comments" not in value: + value["comments"] = "" + if "mp_starcat_time" not in value: + value["mp_starcat_time"] = rows["mp_starcat_time"][0] + if "obsid" not in value: + value["obsid"] = rows["obsid"][0] # sanity checks: # AGASC IDs are not checked to exist because a star could be observed without it being # in the starcheck catalog (a spoiler star), but the observation must exist and have # matching mp_starcat_time and OBSID - if rows['obsid'][0] != value['obsid']: - raise Exception(f'inconsistent observation spec {value}') - if rows['mp_starcat_time'][0] != value['mp_starcat_time']: - raise Exception(f'inconsistent observation spec {value}') + if rows["obsid"][0] != value["obsid"]: + raise Exception(f"inconsistent observation spec {value}") + if rows["mp_starcat_time"][0] != value["mp_starcat_time"]: + raise Exception(f"inconsistent observation spec {value}") rows = [] - for value in status['obs']: - for agasc_id in value['agasc_id']: + for value in status["obs"]: + for agasc_id in value["agasc_id"]: row = value.copy() - row['agasc_id'] = agasc_id + row["agasc_id"] = agasc_id rows.append(row) - status['obs'] = rows + status["obs"] = rows return status @@ -114,7 +114,7 @@ def parse_args( bad_star_source=None, obsid=None, status=None, - comments='', + comments="", agasc_id=None, mp_starcat_time=None, **_, @@ -142,23 +142,23 @@ def parse_args( mags = [] if bad_star_id and bad_star_source is None: - raise RuntimeError('If you specify bad_star, you must specify bad_star_source') + raise RuntimeError("If you specify bad_star, you must specify bad_star_source") if filename is not None: status_file = _parse_obs_status_file(filename) - bad = status_file['bad'] - obs_status_override = status_file['obs'] - mags = status_file['mags'] + bad = status_file["bad"] + obs_status_override = status_file["obs"] + mags = status_file["mags"] if (obsid is not None or mp_starcat_time is not None) and status is not None: row = { - 'mp_starcat_time': mp_starcat_time, - 'agasc_id': agasc_id, - 'obsid': obsid, - 'status': status, - 'comments': comments, + "mp_starcat_time": mp_starcat_time, + "agasc_id": agasc_id, + "obsid": obsid, + "status": status, + "comments": comments, } - optional = ['obsid', 'mp_starcat_time', 'agasc_id', 'comments'] + optional = ["obsid", "mp_starcat_time", "agasc_id", "comments"] obs_status_override.append( { key: row[key] @@ -171,12 +171,12 @@ def parse_args( bad_dict = dict(bad) if bs in bad_dict and bad_dict[bs] != bad_star_source: raise RuntimeError( - 'name collision: conflicting bad_star in file and in args' + "name collision: conflicting bad_star in file and in args" ) if (bs, bad_star_source) not in bad: bad.append((bs, bad_star_source)) - status = {'obs': obs_status_override, 'bad': bad, 'mags': mags} + status = {"obs": obs_status_override, "bad": bad, "mags": mags} status = _sanitize_args(status) return status @@ -189,36 +189,36 @@ def get_obs_status_parser(): """ parser = argparse.ArgumentParser(add_help=False) status = parser.add_argument_group( - 'OBS/star status', + "OBS/star status", ( 'options to modify the "bads" and "obs" tables in AGASC supplement.' - ' Modifications to supplement happen before all magnitude estimates are' - ' made.' + " Modifications to supplement happen before all magnitude estimates are" + " made." ), ) status.add_argument( - '--obs-status-file', + "--obs-status-file", help=( - 'YAML file with star/observation status. ' - 'More info at https://sot.github.io/agasc/supplement.html' + "YAML file with star/observation status. " + "More info at https://sot.github.io/agasc/supplement.html" ), ) status.add_argument( - '--mp-starcat-time', + "--mp-starcat-time", help=( - 'Observation starcat time for status override. ' - 'Usually the mission planning catalog time' + "Observation starcat time for status override. " + "Usually the mission planning catalog time" ), ) - status.add_argument('--obsid', help='OBSID for status override.', type=int) - status.add_argument('--agasc-id', help='AGASC ID for status override.', type=int) - status.add_argument('--status', help='Status to override.') - status.add_argument('--comments', help='Comments for status override.', default='') + status.add_argument("--obsid", help="OBSID for status override.", type=int) + status.add_argument("--agasc-id", help="AGASC ID for status override.", type=int) + status.add_argument("--status", help="Status to override.") + status.add_argument("--comments", help="Comments for status override.", default="") status.add_argument( - '--bad-star-id', - help='AGASC ID of bad star.', + "--bad-star-id", + help="AGASC ID of bad star.", default=[], - action='append', + action="append", type=int, ) status.add_argument( @@ -239,12 +239,12 @@ def get_parser(): ) parse.add_argument( "--output-dir", - default='.', + default=".", type=Path, help="Directory containing agasc_supplement.h5 (default='.')", ) parse.add_argument( - '--log-level', default='info', choices=['debug', 'info', 'warning', 'error'] + "--log-level", default="info", choices=["debug", "info", "warning", "error"] ) parse.add_argument( "--dry-run", @@ -264,25 +264,25 @@ def update(args): """ status = parse_args(filename=args.obs_status_file, **vars(args)) - if status['mags']: + if status["mags"]: update_mags_table( - args.output_dir / 'agasc_supplement.h5', - status['mags'], + args.output_dir / "agasc_supplement.h5", + status["mags"], dry_run=args.dry_run, ) - if status['obs']: + if status["obs"]: update_obs_table( - args.output_dir / 'agasc_supplement.h5', status['obs'], dry_run=args.dry_run + args.output_dir / "agasc_supplement.h5", status["obs"], dry_run=args.dry_run ) - if status['bad']: + if status["bad"]: add_bad_star( - args.output_dir / 'agasc_supplement.h5', status['bad'], dry_run=args.dry_run + args.output_dir / "agasc_supplement.h5", status["bad"], dry_run=args.dry_run ) agasc_ids = sorted( - set([o['agasc_id'] for o in status['obs']] + [o[0] for o in status['bad']]) + set([o["agasc_id"] for o in status["obs"]] + [o[0] for o in status["bad"]]) ) return agasc_ids @@ -293,16 +293,16 @@ def main(): """ import kadi.commands - kadi.commands.conf.commands_version = '1' + kadi.commands.conf.commands_version = "1" args = get_parser().parse_args() - status_to_int = {'ok': 0, 'good': 0, 'bad': 1} + status_to_int = {"ok": 0, "good": 0, "bad": 1} if args.status and args.status.lower() in status_to_int: args.status = status_to_int[args.status.lower()] pyyaks.logger.get_logger( - name='agasc.supplement', + name="agasc.supplement", level=args.log_level.upper(), format="%(asctime)s %(message)s", ) @@ -311,5 +311,5 @@ def main(): update(args) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/supplement/magnitudes/mag_estimate.py b/agasc/supplement/magnitudes/mag_estimate.py index 89bc2f82..f808a193 100644 --- a/agasc/supplement/magnitudes/mag_estimate.py +++ b/agasc/supplement/magnitudes/mag_estimate.py @@ -27,12 +27,12 @@ from agasc import get_star -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") MAX_MAG = 15 MASK = { - 'mouse_bit': np.array( + "mouse_bit": np.array( [ [True, True, True, True, True, True, True, True], [True, True, False, False, False, False, True, True], @@ -48,14 +48,14 @@ EXCEPTION_MSG = { - -1: 'Unknown', - 0: 'OK', - 1: 'No level 0 data', - 2: 'No telemetry data', - 3: 'Mismatch in telemetry between aca_l0 and cheta', - 4: 'Time mismatch between cheta and level0', - 5: 'Failed job', - 6: 'Suspect observation', + -1: "Unknown", + 0: "OK", + 1: "No level 0 data", + 2: "No telemetry data", + 3: "Mismatch in telemetry between aca_l0 and cheta", + 4: "Time mismatch between cheta and level0", + 5: "Failed job", + 6: "Suspect observation", } EXCEPTION_CODES = collections.defaultdict(lambda: -1) EXCEPTION_CODES.update({msg: code for code, msg in EXCEPTION_MSG.items() if code > 0}) @@ -63,7 +63,7 @@ class MagStatsException(Exception): def __init__( - self, msg='', agasc_id=None, obsid=None, mp_starcat_time=None, **kwargs + self, msg="", agasc_id=None, obsid=None, mp_starcat_time=None, **kwargs ): super().__init__(msg) self.error_code = EXCEPTION_CODES[msg] @@ -80,7 +80,7 @@ def __init__( exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is None: - self.exception = {'type': '', 'value': '', 'traceback': ''} + self.exception = {"type": "", "value": "", "traceback": ""} else: stack_trace = [] for step in traceback.extract_tb(exc_traceback): @@ -88,25 +88,25 @@ def __init__( stack_trace.append(f" {step.line}") self.exception = { - 'type': exc_type.__name__, - 'value': str(exc_value), - 'traceback': '\n'.join(stack_trace), + "type": exc_type.__name__, + "value": str(exc_value), + "traceback": "\n".join(stack_trace), } def __str__(self): return ( - f'MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, ' - f'obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})' + f"MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, " + f"obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})" ) def __iter__(self): - yield 'error_code', self.error_code - yield 'msg', self.msg - yield 'agasc_id', self.agasc_id - yield 'obsid', self.obsid - yield 'mp_starcat_time', self.mp_starcat_time - yield 'exception_type', self.exception['type'] - yield 'traceback', self.exception['traceback'] + yield "error_code", self.error_code + yield "msg", self.msg + yield "agasc_id", self.agasc_id + yield "obsid", self.obsid + yield "mp_starcat_time", self.mp_starcat_time + yield "exception_type", self.exception["type"] + yield "traceback", self.exception["traceback"] def _magnitude_correction(time, mag_aca): @@ -128,8 +128,8 @@ def _magnitude_correction(time, mag_aca): ], } - q = params['p'] - t_ref = DateTime(params['t_ref']) + q = params["p"] + t_ref = DateTime(params["t_ref"]) dmag = q[0] + (q[1] + q[2] * np.atleast_1d(time)) * np.exp( q[3] + q[4] * np.atleast_1d(mag_aca) ) @@ -236,7 +236,7 @@ def get_star_position(star, telem): rad_to_arcsec = 206264.81 q = np.array( - [telem['AOATTQT1'], telem['AOATTQT2'], telem['AOATTQT3'], telem['AOATTQT4']] + [telem["AOATTQT1"], telem["AOATTQT2"], telem["AOATTQT3"], telem["AOATTQT4"]] ).transpose() norm = np.sum(q**2, axis=1, keepdims=True) # I am just normalizing q, just in case. @@ -247,7 +247,7 @@ def get_star_position(star, telem): q_att = Quat(q=q) ts = q_att.transform - star_pos_eci = Ska.quatutil.radec2eci(star['RA_PMCORR'], star['DEC_PMCORR']) + star_pos_eci = Ska.quatutil.radec2eci(star["RA_PMCORR"], star["DEC_PMCORR"]) d_aca = np.dot( np.dot(aca_misalign, ts.transpose(0, 2, 1)), star_pos_eci ).transpose() @@ -256,42 +256,42 @@ def get_star_position(star, telem): logger.debug( f' star position. AGASC_ID={star["AGASC_ID"]}, ' - f'{len(yag)} samples, ({yag[0]}, {zag[0]})...' + f"{len(yag)} samples, ({yag[0]}, {zag[0]})..." ) return { - 'yang_star': yag, - 'zang_star': zag, + "yang_star": yag, + "zang_star": zag, } # this is in case one has to return empty telemetry _telem_dtype = [ - ('times', 'float64'), - ('IMGSIZE', 'int32'), - ('IMGROW0', 'int16'), - ('IMGCOL0', 'int16'), - ('IMGRAW', 'float32'), - ('AOACASEQ', ' 1: - obs = obs.loc['mp_starcat_time', sorted(obs['mp_starcat_time'])] + obs = obs.loc["mp_starcat_time", sorted(obs["mp_starcat_time"])] telem = [] for i, o in enumerate(obs): try: t = Table(get_telemetry(o)) - t['obsid'] = o['obsid'] - t['agasc_id'] = agasc_id + t["obsid"] = o["obsid"] + t["agasc_id"] = agasc_id telem.append(t) except Exception: if not ignore_exceptions: logger.info(f'{agasc_id=}, obsid={o["obsid"]} failed') exc_type, exc_value, exc_traceback = sys.exc_info() trace = traceback.extract_tb(exc_traceback) - logger.info(f'{exc_type.__name__} {exc_value}') + logger.info(f"{exc_type.__name__} {exc_value}") for step in trace: - logger.info(f' in {step.filename}:{step.lineno}/{step.name}:') - logger.info(f' {step.line}') + logger.info(f" in {step.filename}:{step.lineno}/{step.name}:") + logger.info(f" {step.line}") raise return vstack(telem) @@ -523,38 +523,38 @@ def add_obs_info(telem, obs_stats): The result of calc_obs_stats. :return: """ - logger.debug(' Adding observation info to telemetry...') - obs_stats['obs_ok'] = ( - (obs_stats['n'] > 10) - & (obs_stats['f_mag_est_ok'] > 0.3) - & (obs_stats['lf_variability_100s'] < 1) + logger.debug(" Adding observation info to telemetry...") + obs_stats["obs_ok"] = ( + (obs_stats["n"] > 10) + & (obs_stats["f_mag_est_ok"] > 0.3) + & (obs_stats["lf_variability_100s"] < 1) ) - obs_stats['comments'] = np.zeros(len(obs_stats), dtype=' 0 - and np.isfinite(s['q75']) - and np.isfinite(s['q25']) + np.any(telem["mag_est_ok"][o]) + and s["f_mag_est_ok"] > 0 + and np.isfinite(s["q75"]) + and np.isfinite(s["q25"]) ): - iqr = s['q75'] - s['q25'] - telem['obs_outlier'][o] = ( - telem[o]['mag_est_ok'] + iqr = s["q75"] - s["q25"] + telem["obs_outlier"][o] = ( + telem[o]["mag_est_ok"] & (iqr > 0) & ( - (telem[o]['mags'] < s['q25'] - 1.5 * iqr) - | (telem[o]['mags'] > s['q75'] + 1.5 * iqr) + (telem[o]["mags"] < s["q25"] - 1.5 * iqr) + | (telem[o]["mags"] > s["q75"] + 1.5 * iqr) ) ) - logger.debug(f' Adding observation info to telemetry {obsid=}') + logger.debug(f" Adding observation info to telemetry {obsid=}") return telem @@ -578,20 +578,20 @@ def get_mag_from_img(slot_data, t_start, ok=True): Only magnitudes for entries with ok=True are calculated. The rest are set to MAX_MAG. :return: """ - logger.debug(' magnitude from images...') + logger.debug(" magnitude from images...") dark_cal = get_dark_cal_image( t_start, - 'nearest', - t_ccd_ref=np.mean(slot_data['TEMPCCD'] - 273.16), + "nearest", + t_ccd_ref=np.mean(slot_data["TEMPCCD"] - 273.16), aca_image=False, ) # all images will be 8x8, with a centered mask, imgrow will always be the one of the 8x8 corner. imgrow_8x8 = np.where( - slot_data['IMGSIZE'] == 8, slot_data['IMGROW0'], slot_data['IMGROW0'] - 1 + slot_data["IMGSIZE"] == 8, slot_data["IMGROW0"], slot_data["IMGROW0"] - 1 ) imgcol_8x8 = np.where( - slot_data['IMGSIZE'] == 8, slot_data['IMGCOL0'], slot_data['IMGCOL0'] - 1 + slot_data["IMGSIZE"] == 8, slot_data["IMGCOL0"], slot_data["IMGCOL0"] - 1 ) # subtract closest dark cal @@ -599,8 +599,8 @@ def get_mag_from_img(slot_data, t_start, ok=True): staggered_aca_slice( dark_cal.astype(float), dark, 512 + imgrow_8x8, 512 + imgcol_8x8 ) - img_sub = slot_data['IMGRAW'] - dark * 1.696 / 5 - img_sub.mask |= MASK['mouse_bit'] + img_sub = slot_data["IMGRAW"] - dark * 1.696 / 5 + img_sub.mask |= MASK["mouse_bit"] # calculate magnitude mag = np.ones(len(slot_data)) * MAX_MAG @@ -610,7 +610,7 @@ def get_mag_from_img(slot_data, t_start, ok=True): mag[mag > MAX_MAG] = MAX_MAG # this extra step is to investigate the background scale dark = np.ma.array(dark * 1.696 / 5, mask=img_sub.mask) - img_raw = np.ma.array(slot_data['IMGRAW'], mask=img_sub.mask) + img_raw = np.ma.array(slot_data["IMGRAW"], mask=img_sub.mask) dark_count = np.ma.sum(np.ma.sum(dark, axis=1), axis=1) img_count = np.ma.sum(np.ma.sum(img_raw, axis=1), axis=1) @@ -618,11 +618,11 @@ def get_mag_from_img(slot_data, t_start, ok=True): yag = np.zeros(len(slot_data)) zag = np.zeros(len(slot_data)) pixel_center = np.arange(8) + 0.5 - projected_image = np.ma.sum(slot_data['IMGRAW'], axis=1) + projected_image = np.ma.sum(slot_data["IMGRAW"], axis=1) col = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum( projected_image, axis=1 ) - projected_image = np.ma.sum(slot_data['IMGRAW'], axis=2) + projected_image = np.ma.sum(slot_data["IMGRAW"], axis=2) row = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum( projected_image, axis=1 ) @@ -630,115 +630,115 @@ def get_mag_from_img(slot_data, t_start, ok=True): y_pixel = row + imgrow_8x8 z_pixel = col + imgcol_8x8 yag[m], zag[m] = pixels_to_yagzag(y_pixel[m], z_pixel[m]) - logger.debug(f' magnitude from images... {len(mag)} samples: {mag[0]:.2f}...') + logger.debug(f" magnitude from images... {len(mag)} samples: {mag[0]:.2f}...") return { - 'mags_img': mag, - 'yang_img': yag, - 'zang_img': zag, - 'counts_img': img_count, - 'counts_dark': dark_count, + "mags_img": mag, + "yang_img": yag, + "zang_img": zag, + "counts_img": img_count, + "counts_dark": dark_count, } OBS_STATS_INFO = { - 'agasc_id': 'AGASC ID of the star', - 'obsid': 'OBSID at the time the star catalog is commanded (from kadi.commands)', - 'slot': 'Slot number', - 'type': 'GUI/ACQ/BOT', - 'mp_starcat_time': ( - 'The time at which the star catalog is commanded (from kadi.commands)' + "agasc_id": "AGASC ID of the star", + "obsid": "OBSID at the time the star catalog is commanded (from kadi.commands)", + "slot": "Slot number", + "type": "GUI/ACQ/BOT", + "mp_starcat_time": ( + "The time at which the star catalog is commanded (from kadi.commands)" ), - 'tstart': ( - 'Start time of the observation according to kadi.commands (in cxc seconds)' + "tstart": ( + "Start time of the observation according to kadi.commands (in cxc seconds)" ), - 'tstop': ( - 'Start time of the observation according to kadi.commands (in cxc seconds)' + "tstop": ( + "Start time of the observation according to kadi.commands (in cxc seconds)" ), - 'mag_correction': 'Overall correction applied to the magnitude estimate', - 'responsivity': 'Responsivity correction applied to the magnitude estimate', - 'droop_shift': 'Droop shift correction applied to the magnitude estimate', - 'mag_aca': 'ACA star magnitude from the AGASC catalog', - 'mag_aca_err': 'ACA star magnitude uncertainty from the AGASC catalog', - 'row': ( - 'Expected row number, based on star location and yanf/zang from' - ' mica.archive.starcheck DB' + "mag_correction": "Overall correction applied to the magnitude estimate", + "responsivity": "Responsivity correction applied to the magnitude estimate", + "droop_shift": "Droop shift correction applied to the magnitude estimate", + "mag_aca": "ACA star magnitude from the AGASC catalog", + "mag_aca_err": "ACA star magnitude uncertainty from the AGASC catalog", + "row": ( + "Expected row number, based on star location and yanf/zang from" + " mica.archive.starcheck DB" ), - 'col': ( - 'Expected col number, based on star location and yanf/zang from' - ' mica.archive.starcheck DB' + "col": ( + "Expected col number, based on star location and yanf/zang from" + " mica.archive.starcheck DB" ), - 'mag_img': 'Magnitude estimate from image telemetry (uncorrected)', - 'mag_obs': 'Estimated ACA star magnitude', - 'mag_obs_err': 'Estimated ACA star magnitude uncertainty', - 'aoacmag_mean': 'Mean of AOACMAG from telemetry', - 'aoacmag_err': 'Standard deviation of AOACMAG from telemetry', - 'aoacmag_q25': '1st quartile of AOACMAG from telemetry', - 'aoacmag_median': 'Median of AOACMAG from telemetry', - 'aoacmag_q75': '3rd quartile of AOACMAG from telemetry', - 'counts_img': 'Raw counts from image telemetry, summed over the mouse-bit window', - 'counts_dark': 'Expected counts from background, summed over the mouse-bit window', - 'f_kalman': ( + "mag_img": "Magnitude estimate from image telemetry (uncorrected)", + "mag_obs": "Estimated ACA star magnitude", + "mag_obs_err": "Estimated ACA star magnitude uncertainty", + "aoacmag_mean": "Mean of AOACMAG from telemetry", + "aoacmag_err": "Standard deviation of AOACMAG from telemetry", + "aoacmag_q25": "1st quartile of AOACMAG from telemetry", + "aoacmag_median": "Median of AOACMAG from telemetry", + "aoacmag_q75": "3rd quartile of AOACMAG from telemetry", + "counts_img": "Raw counts from image telemetry, summed over the mouse-bit window", + "counts_dark": "Expected counts from background, summed over the mouse-bit window", + "f_kalman": ( 'Fraction of all samples where AOACASEQ == "KALM" and AOPCADMD == "NPNT"' - ' (n_kalman/n)' + " (n_kalman/n)" ), - 'f_track': 'Fraction of kalman samples with AOACFCT == "TRAK" (n_track/n_kalman)', - 'f_mag_est_ok': ( - 'Fraction of all samples included in magnitude estimate regardless of centroid' - ' residual (n_mag_est_ok/n_kalman)' + "f_track": 'Fraction of kalman samples with AOACFCT == "TRAK" (n_track/n_kalman)', + "f_mag_est_ok": ( + "Fraction of all samples included in magnitude estimate regardless of centroid" + " residual (n_mag_est_ok/n_kalman)" ), - 'f_mag_est_ok_3': ( - 'Fraction of kalman samples with (mag_est_ok & dr3) == True (n_ok_3/n_kalman)' + "f_mag_est_ok_3": ( + "Fraction of kalman samples with (mag_est_ok & dr3) == True (n_ok_3/n_kalman)" ), - 'f_mag_est_ok_5': ( - 'Fraction of kalman samples with (mag_est_ok & dbox5) == True (n_ok_5/n_kalman)' + "f_mag_est_ok_5": ( + "Fraction of kalman samples with (mag_est_ok & dbox5) == True (n_ok_5/n_kalman)" ), - 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5.', # fix this - 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an + "f_ok": "n_ok_5 / n_kalman. Same as f_ok_5.", # fix this + "f_ok_3": """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", - 'f_ok_5': """ + "f_ok_5": """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all.""", - 'f_dr3': ( - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' - ' (n_dr3/n_mag_est_ok)' + "f_dr3": ( + "Fraction of mag-est-ok samples with centroid residual < 3 arcsec" + " (n_dr3/n_mag_est_ok)" ), - 'f_dbox5': ( - 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box ' - '(n_dbox5/n_mag_est_ok)' + "f_dbox5": ( + "Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box " + "(n_dbox5/n_mag_est_ok)" ), - 'q25': '1st quartile of estimated magnitude', - 'median': 'Median of estimated magnitude', - 'q75': '1st quartile of estimated magnitude', - 'mean': 'Mean of estimated magnitude', - 'mean_err': 'Uncertainty in the mean of estimated magnitude', - 'std': 'Standard deviation of estimated magnitude', - 'skew': 'Skewness of estimated magnitude', - 'kurt': 'Kurtosis of estimated magnitude', - 't_mean': 'Mean of estimated magnitude after removing outliers', - 't_mean_err': ( - 'Uncertainty in the mean of estimated magnitude after removing outliers' + "q25": "1st quartile of estimated magnitude", + "median": "Median of estimated magnitude", + "q75": "1st quartile of estimated magnitude", + "mean": "Mean of estimated magnitude", + "mean_err": "Uncertainty in the mean of estimated magnitude", + "std": "Standard deviation of estimated magnitude", + "skew": "Skewness of estimated magnitude", + "kurt": "Kurtosis of estimated magnitude", + "t_mean": "Mean of estimated magnitude after removing outliers", + "t_mean_err": ( + "Uncertainty in the mean of estimated magnitude after removing outliers" ), - 't_std': 'Standard deviation of estimated magnitude after removing outliers', - 't_skew': 'Skewness of estimated magnitude after removing outliers', - 't_kurt': 'Kurtosis of estimated magnitude after removing outliers', - 'n': 'Number of samples', - 'n_ok': 'Number of samples with (kalman & mag_est_ok & dbox5) == True', - 'outliers': 'Number of outliers (+- 3 IQR)', - 'lf_variability_100s': 'Rolling mean of OK magnitudes with a 100 second window', - 'lf_variability_500s': 'Rolling mean of OK magnitudes with a 500 second window', - 'lf_variability_1000s': 'Rolling mean of OK magnitudes with a 1000 second window', - 'tempccd': 'CCD temperature', - 'dr_star': 'Angle residual', - 'obs_ok': 'Boolean flag: everything OK with this observation', - 'obs_suspect': 'Boolean flag: this observation is "suspect"', - 'obs_fail': ( - 'Boolean flag: a processing error when estimating magnitude for this' - ' observation' + "t_std": "Standard deviation of estimated magnitude after removing outliers", + "t_skew": "Skewness of estimated magnitude after removing outliers", + "t_kurt": "Kurtosis of estimated magnitude after removing outliers", + "n": "Number of samples", + "n_ok": "Number of samples with (kalman & mag_est_ok & dbox5) == True", + "outliers": "Number of outliers (+- 3 IQR)", + "lf_variability_100s": "Rolling mean of OK magnitudes with a 100 second window", + "lf_variability_500s": "Rolling mean of OK magnitudes with a 500 second window", + "lf_variability_1000s": "Rolling mean of OK magnitudes with a 1000 second window", + "tempccd": "CCD temperature", + "dr_star": "Angle residual", + "obs_ok": "Boolean flag: everything OK with this observation", + "obs_suspect": 'Boolean flag: this observation is "suspect"', + "obs_fail": ( + "Boolean flag: a processing error when estimating magnitude for this" + " observation" ), - 'comments': '', - 'w': 'Weight to be used on a weighted mean (1/std)', - 'mean_corrected': 'Corrected mean used in weighted mean (t_mean + mag_correction)', - 'weighted_mean': 'Mean weighted by inverse of standard deviation (mean/std)', + "comments": "", + "w": "Weight to be used on a weighted mean (1/std)", + "mean_corrected": "Corrected mean used in weighted mean (t_mean + mag_correction)", + "weighted_mean": "Mean weighted by inverse of standard deviation (mean/std)", } @@ -761,79 +761,79 @@ def get_obs_stats(obs, telem=None): star_obs_catalogs.load() - star = get_star(obs['agasc_id'], use_supplement=False) - start = CxoTime(obs['obs_start']).cxcsec - stop = CxoTime(obs['obs_stop']).cxcsec + star = get_star(obs["agasc_id"], use_supplement=False) + start = CxoTime(obs["obs_start"]).cxcsec + stop = CxoTime(obs["obs_stop"]).cxcsec stats = { - k: obs[k] for k in ['agasc_id', 'obsid', 'slot', 'type', 'mp_starcat_time'] + k: obs[k] for k in ["agasc_id", "obsid", "slot", "type", "mp_starcat_time"] } - stats['mp_starcat_time'] = stats['mp_starcat_time'] - droop_shift = get_droop_systematic_shift(star['MAG_ACA']) + stats["mp_starcat_time"] = stats["mp_starcat_time"] + droop_shift = get_droop_systematic_shift(star["MAG_ACA"]) responsivity = get_responsivity(start) stats.update( { - 'tstart': start, - 'tstop': stop, - 'mag_correction': -responsivity - droop_shift, - 'responsivity': responsivity, - 'droop_shift': droop_shift, - 'mag_aca': star['MAG_ACA'], - 'mag_aca_err': star['MAG_ACA_ERR'] / 100, - 'row': obs['row'], - 'col': obs['col'], + "tstart": start, + "tstop": stop, + "mag_correction": -responsivity - droop_shift, + "responsivity": responsivity, + "droop_shift": droop_shift, + "mag_aca": star["MAG_ACA"], + "mag_aca_err": star["MAG_ACA_ERR"] / 100, + "row": obs["row"], + "col": obs["col"], } ) # other default values stats.update( { - 'mag_img': np.inf, - 'mag_obs': np.inf, - 'mag_obs_err': np.inf, - 'aoacmag_mean': np.inf, - 'aoacmag_err': np.inf, - 'aoacmag_q25': np.inf, - 'aoacmag_median': np.inf, - 'aoacmag_q75': np.inf, - 'counts_img': np.inf, - 'counts_dark': np.inf, - 'f_kalman': 0.0, - 'f_track': 0.0, - 'f_dbox5': 0.0, - 'f_dr3': 0.0, - 'f_mag_est_ok': 0.0, - 'f_mag_est_ok_3': 0.0, - 'f_mag_est_ok_5': 0.0, - 'f_ok': 0.0, - 'f_ok_3': 0.0, - 'f_ok_5': 0.0, - 'q25': np.inf, - 'median': np.inf, - 'q75': np.inf, - 'mean': np.inf, - 'mean_err': np.inf, - 'std': np.inf, - 'skew': np.inf, - 'kurt': np.inf, - 't_mean': np.inf, - 't_mean_err': np.inf, - 't_std': np.inf, - 't_skew': np.inf, - 't_kurt': np.inf, - 'n': 0, - 'n_ok': 0, - 'n_ok_3': 0, - 'n_ok_5': 0, - 'n_mag_est_ok': 0, - 'n_mag_est_ok_3': 0, - 'n_mag_est_ok_5': 0, - 'outliers': -1, - 'lf_variability_100s': np.inf, - 'lf_variability_500s': np.inf, - 'lf_variability_1000s': np.inf, - 'tempccd': np.nan, - 'dr_star': np.inf, + "mag_img": np.inf, + "mag_obs": np.inf, + "mag_obs_err": np.inf, + "aoacmag_mean": np.inf, + "aoacmag_err": np.inf, + "aoacmag_q25": np.inf, + "aoacmag_median": np.inf, + "aoacmag_q75": np.inf, + "counts_img": np.inf, + "counts_dark": np.inf, + "f_kalman": 0.0, + "f_track": 0.0, + "f_dbox5": 0.0, + "f_dr3": 0.0, + "f_mag_est_ok": 0.0, + "f_mag_est_ok_3": 0.0, + "f_mag_est_ok_5": 0.0, + "f_ok": 0.0, + "f_ok_3": 0.0, + "f_ok_5": 0.0, + "q25": np.inf, + "median": np.inf, + "q75": np.inf, + "mean": np.inf, + "mean_err": np.inf, + "std": np.inf, + "skew": np.inf, + "kurt": np.inf, + "t_mean": np.inf, + "t_mean_err": np.inf, + "t_std": np.inf, + "t_skew": np.inf, + "t_kurt": np.inf, + "n": 0, + "n_ok": 0, + "n_ok_3": 0, + "n_ok_5": 0, + "n_mag_est_ok": 0, + "n_mag_est_ok_3": 0, + "n_mag_est_ok_5": 0, + "outliers": -1, + "lf_variability_100s": np.inf, + "lf_variability_500s": np.inf, + "lf_variability_1000s": np.inf, + "tempccd": np.nan, + "dr_star": np.inf, } ) @@ -868,18 +868,18 @@ def calc_obs_stats(telem): # From here on, we only consider the telemetry in NPNT in Kalman mode. All # subsequent counts and fractions are calculated from this subset. ########################################################################### - telem = telem[(telem['AOACASEQ'] == 'KALM') & (telem['AOPCADMD'] == 'NPNT')] - times = telem['times'] + telem = telem[(telem["AOACASEQ"] == "KALM") & (telem["AOPCADMD"] == "NPNT")] + times = telem["times"] - dr3 = telem['dr'] < 3 - dbox5 = (np.abs(telem['dy']) < 5) & (np.abs(telem['dz']) < 5) + dr3 = telem["dr"] < 3 + dbox5 = (np.abs(telem["dy"]) < 5) & (np.abs(telem["dz"]) < 5) # Samples used in the magnitude estimation processing # note that SP flag is not included - mag_est_ok = (telem['AOACIIR'] == 'OK') & (telem['AOACFCT'] == 'TRAK') - aca_trak = telem['AOACFCT'] == 'TRAK' - sat_pix = telem['AOACISP'] == 'OK' - ion_rad = telem['AOACIIR'] == 'OK' + mag_est_ok = (telem["AOACIIR"] == "OK") & (telem["AOACFCT"] == "TRAK") + aca_trak = telem["AOACFCT"] == "TRAK" + sat_pix = telem["AOACISP"] == "OK" + ion_rad = telem["AOACIIR"] == "OK" n_kalman = len(telem) f_kalman = n_kalman / n_total @@ -896,40 +896,40 @@ def calc_obs_stats(telem): # Select readouts OK for OBC Kalman filter and compute star mean offset ok = mag_est_ok & dbox5 if np.any(ok): - yang_mean = np.mean(telem['yang_img'][ok] - telem['yang_star'][ok]) - zang_mean = np.mean(telem['zang_img'][ok] - telem['zang_star'][ok]) + yang_mean = np.mean(telem["yang_img"][ok] - telem["yang_star"][ok]) + zang_mean = np.mean(telem["zang_img"][ok] - telem["zang_star"][ok]) dr_star = np.sqrt(yang_mean**2 + zang_mean**2) else: dr_star = np.inf stats = { - 'f_kalman': f_kalman, - 'f_track': (np.count_nonzero(aca_trak) / n_kalman) if n_kalman else 0, - 'f_dbox5': f_5, - 'f_dr3': f_3, - 'f_mag_est_ok': (n_mag_est_ok / n_kalman) if n_kalman else 0, - 'f_mag_est_ok_3': (n_mag_est_ok_3 / n_kalman) if n_kalman else 0, - 'f_mag_est_ok_5': (n_mag_est_ok_5 / n_kalman) if n_kalman else 0, - 'f_ok': (n_ok_5 / n_kalman) if n_kalman else 0, - 'f_ok_3': (n_ok_3 / n_kalman) if n_kalman else 0, - 'f_ok_5': (n_ok_5 / n_kalman) if n_kalman else 0, - 'n': n_total, - 'n_ok': n_ok_5, - 'n_ok_5': n_ok_5, - 'n_ok_3': n_ok_3, - 'n_mag_est_ok': n_mag_est_ok, - 'n_mag_est_ok_3': n_mag_est_ok_3, - 'n_mag_est_ok_5': n_mag_est_ok_5, - 'dr_star': dr_star, + "f_kalman": f_kalman, + "f_track": (np.count_nonzero(aca_trak) / n_kalman) if n_kalman else 0, + "f_dbox5": f_5, + "f_dr3": f_3, + "f_mag_est_ok": (n_mag_est_ok / n_kalman) if n_kalman else 0, + "f_mag_est_ok_3": (n_mag_est_ok_3 / n_kalman) if n_kalman else 0, + "f_mag_est_ok_5": (n_mag_est_ok_5 / n_kalman) if n_kalman else 0, + "f_ok": (n_ok_5 / n_kalman) if n_kalman else 0, + "f_ok_3": (n_ok_3 / n_kalman) if n_kalman else 0, + "f_ok_5": (n_ok_5 / n_kalman) if n_kalman else 0, + "n": n_total, + "n_ok": n_ok_5, + "n_ok_5": n_ok_5, + "n_ok_3": n_ok_3, + "n_mag_est_ok": n_mag_est_ok, + "n_mag_est_ok_3": n_mag_est_ok_3, + "n_mag_est_ok_5": n_mag_est_ok_5, + "dr_star": dr_star, } - if stats['n_mag_est_ok_3'] < 10: + if stats["n_mag_est_ok_3"] < 10: return stats aoacmag_q25, aoacmag_q50, aoacmag_q75 = np.quantile( - telem['AOACMAG'][ok], [0.25, 0.5, 0.75] + telem["AOACMAG"][ok], [0.25, 0.5, 0.75] ) - mags = telem['mags'] + mags = telem["mags"] q25, q50, q75 = np.quantile(mags[ok], [0.25, 0.5, 0.75]) iqr = q75 - q25 outlier = ok & ((mags > q75 + 3 * iqr) | (mags < q25 - 3 * iqr)) @@ -944,39 +944,39 @@ def calc_obs_stats(telem): stats.update( { - 'aoacmag_mean': np.mean(telem['AOACMAG'][ok]), - 'aoacmag_err': np.std(telem['AOACMAG'][ok]), - 'aoacmag_q25': aoacmag_q25, - 'aoacmag_median': aoacmag_q50, - 'aoacmag_q75': aoacmag_q75, - 'q25': q25, - 'median': q50, - 'q75': q75, - 'counts_img': np.mean(telem['counts_img'][ok]), - 'counts_dark': np.mean(telem['counts_dark'][ok]), - 'mean': np.mean(mags[ok]), - 'mean_err': scipy.stats.sem(mags[ok]), - 'std': np.std(mags[ok]), - 'skew': scipy.stats.skew(mags), - 'kurt': scipy.stats.kurtosis(mags), - 't_mean': np.mean(mags[ok & (~outlier)]), - 't_mean_err': scipy.stats.sem(mags[ok & (~outlier)]), - 't_std': np.std(mags[ok & (~outlier)]), - 't_skew': scipy.stats.skew(mags[ok & (~outlier)]), - 't_kurt': scipy.stats.kurtosis(mags[ok & (~outlier)]), - 'outliers': np.count_nonzero(outlier), - 'lf_variability_100s': np.max(s_100s) - np.min(s_100s), - 'lf_variability_500s': np.max(s_500s) - np.min(s_500s), - 'lf_variability_1000s': np.max(s_1000s) - np.min(s_1000s), - 'tempccd': np.mean(telem['TEMPCCD'][ok]) - 273.16, + "aoacmag_mean": np.mean(telem["AOACMAG"][ok]), + "aoacmag_err": np.std(telem["AOACMAG"][ok]), + "aoacmag_q25": aoacmag_q25, + "aoacmag_median": aoacmag_q50, + "aoacmag_q75": aoacmag_q75, + "q25": q25, + "median": q50, + "q75": q75, + "counts_img": np.mean(telem["counts_img"][ok]), + "counts_dark": np.mean(telem["counts_dark"][ok]), + "mean": np.mean(mags[ok]), + "mean_err": scipy.stats.sem(mags[ok]), + "std": np.std(mags[ok]), + "skew": scipy.stats.skew(mags), + "kurt": scipy.stats.kurtosis(mags), + "t_mean": np.mean(mags[ok & (~outlier)]), + "t_mean_err": scipy.stats.sem(mags[ok & (~outlier)]), + "t_std": np.std(mags[ok & (~outlier)]), + "t_skew": scipy.stats.skew(mags[ok & (~outlier)]), + "t_kurt": scipy.stats.kurtosis(mags[ok & (~outlier)]), + "outliers": np.count_nonzero(outlier), + "lf_variability_100s": np.max(s_100s) - np.min(s_100s), + "lf_variability_500s": np.max(s_500s) - np.min(s_500s), + "lf_variability_1000s": np.max(s_1000s) - np.min(s_1000s), + "tempccd": np.mean(telem["TEMPCCD"][ok]) - 273.16, } ) stats.update( { - 'mag_img': np.mean(telem['mags_img'][ok & (~outlier)]), - 'mag_obs': stats['t_mean'], - 'mag_obs_err': stats['t_mean_err'], + "mag_img": np.mean(telem["mags_img"][ok & (~outlier)]), + "mag_obs": stats["t_mean"], + "mag_obs_err": stats["t_mean_err"], } ) @@ -984,145 +984,145 @@ def calc_obs_stats(telem): AGASC_ID_STATS_INFO = { - 'last_obs_time': ( - 'CXC seconds corresponding to the last mp_starcat_time for the star' + "last_obs_time": ( + "CXC seconds corresponding to the last mp_starcat_time for the star" ), - 'agasc_id': 'AGASC ID of the star', - 'mag_aca': 'ACA star magnitude from the AGASC catalog', - 'mag_aca_err': 'ACA star magnitude uncertainty from the AGASC catalog', - 'mag_obs': 'Estimated ACA star magnitude', - 'mag_obs_err': 'Estimated ACA star magnitude uncertainty', - 'mag_obs_std': 'Estimated ACA star magnitude standard deviation', - 'color': 'Star color from the AGASC catalog', - 'n_obsids': 'Number of observations for the star', - 'n_obsids_fail': 'Number of observations which give an unexpected error', - 'n_obsids_suspect': ( + "agasc_id": "AGASC ID of the star", + "mag_aca": "ACA star magnitude from the AGASC catalog", + "mag_aca_err": "ACA star magnitude uncertainty from the AGASC catalog", + "mag_obs": "Estimated ACA star magnitude", + "mag_obs_err": "Estimated ACA star magnitude uncertainty", + "mag_obs_std": "Estimated ACA star magnitude standard deviation", + "color": "Star color from the AGASC catalog", + "n_obsids": "Number of observations for the star", + "n_obsids_fail": "Number of observations which give an unexpected error", + "n_obsids_suspect": ( 'Number of observations deemed "suspect" and ignored in the magnitude estimate' ), - 'n_obsids_ok': 'Number of observations considered in the magnitude estimate', - 'n_no_mag': 'Number of observations where the star magnitude was not estimated', - 'n': 'Total number of image samples for the star', - 'n_mag_est_ok': ( - 'Total number of image samples included in magnitude estimate for the star' + "n_obsids_ok": "Number of observations considered in the magnitude estimate", + "n_no_mag": "Number of observations where the star magnitude was not estimated", + "n": "Total number of image samples for the star", + "n_mag_est_ok": ( + "Total number of image samples included in magnitude estimate for the star" ), - 'f_mag_est_ok': ( - 'Fraction of all samples included in magnitude estimate regardless of centroid' - ' residual (n_mag_est_ok / n_kalman)' + "f_mag_est_ok": ( + "Fraction of all samples included in magnitude estimate regardless of centroid" + " residual (n_mag_est_ok / n_kalman)" ), - 'f_mag_est_ok_3': ( - 'Fraction of kalman samples that are included in magnitude estimate and within' - ' 3 arcsec (n_mag_est_ok_3 / n_kalman)' + "f_mag_est_ok_3": ( + "Fraction of kalman samples that are included in magnitude estimate and within" + " 3 arcsec (n_mag_est_ok_3 / n_kalman)" ), - 'f_mag_est_ok_5': ( - 'Fraction of kalman samples that are included in magnitude estimate and within' - ' a 5 arcsec box (n_mag_est_ok_5 / n_kalman)' + "f_mag_est_ok_5": ( + "Fraction of kalman samples that are included in magnitude estimate and within" + " a 5 arcsec box (n_mag_est_ok_5 / n_kalman)" ), - 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5.', - 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an + "f_ok": "n_ok_5 / n_kalman. Same as f_ok_5.", + "f_ok_3": """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", - 'f_ok_5': """ + "f_ok_5": """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all.""", - 'median': 'Median magnitude over mag-est-ok samples', - 'sigma_minus': '15.8% quantile of magnitude over mag-est-ok samples', - 'sigma_plus': '84.2% quantile of magnitude over mag-est-ok samples', - 'mean': 'Mean of magnitude over mag-est-ok samples', - 'std': 'Standard deviation of magnitude over mag-est-ok samples', - 'mag_weighted_mean': ( - 'Average of magnitudes over observations, weighed by the inverse of its' - ' standard deviation' + "median": "Median magnitude over mag-est-ok samples", + "sigma_minus": "15.8% quantile of magnitude over mag-est-ok samples", + "sigma_plus": "84.2% quantile of magnitude over mag-est-ok samples", + "mean": "Mean of magnitude over mag-est-ok samples", + "std": "Standard deviation of magnitude over mag-est-ok samples", + "mag_weighted_mean": ( + "Average of magnitudes over observations, weighed by the inverse of its" + " standard deviation" ), - 'mag_weighted_std': 'Uncertainty in the weighted magnitude mean', - 't_mean': 'Mean magnitude after removing outliers on a per-observation basis', - 't_std': ( - 'Magnitude standard deviation after removing outliers on a per-observation' - ' basis' + "mag_weighted_std": "Uncertainty in the weighted magnitude mean", + "t_mean": "Mean magnitude after removing outliers on a per-observation basis", + "t_std": ( + "Magnitude standard deviation after removing outliers on a per-observation" + " basis" ), - 'n_outlier': 'Number of outliers, removed on a per-observation basis', - 't_mean_1': 'Mean magnitude after removing 1.5*IQR outliers', - 't_std_1': 'Magnitude standard deviation after removing 1.5*IQR outliers', - 'n_outlier_1': 'Number of 1.5*IQR outliers', - 't_mean_2': 'Mean magnitude after removing 3*IQR outliers', - 't_std_2': 'Magnitude standard deviation after removing 3*IQR outliers', - 'n_outlier_2': 'Number of 3*IQR outliers', - 'selected_atol': 'abs(mag_obs - mag_aca) > 0.3', - 'selected_rtol': 'abs(mag_obs - mag_aca) > 3 * mag_aca_err', - 'selected_mag_aca_err': 'mag_aca_err > 0.2', - 'selected_color': '(color == 1.5) | (color == 0.7)', - 't_mean_dr3': ( - 'Truncated mean magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "n_outlier": "Number of outliers, removed on a per-observation basis", + "t_mean_1": "Mean magnitude after removing 1.5*IQR outliers", + "t_std_1": "Magnitude standard deviation after removing 1.5*IQR outliers", + "n_outlier_1": "Number of 1.5*IQR outliers", + "t_mean_2": "Mean magnitude after removing 3*IQR outliers", + "t_std_2": "Magnitude standard deviation after removing 3*IQR outliers", + "n_outlier_2": "Number of 3*IQR outliers", + "selected_atol": "abs(mag_obs - mag_aca) > 0.3", + "selected_rtol": "abs(mag_obs - mag_aca) > 3 * mag_aca_err", + "selected_mag_aca_err": "mag_aca_err > 0.2", + "selected_color": "(color == 1.5) | (color == 0.7)", + "t_mean_dr3": ( + "Truncated mean magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 't_std_dr3': ( - 'Truncated magnitude standard deviation after removing outliers and samples' - ' with centroid residual > 3 arcsec on a per-observation basis' + "t_std_dr3": ( + "Truncated magnitude standard deviation after removing outliers and samples" + " with centroid residual > 3 arcsec on a per-observation basis" ), - 'mean_dr3': ( - 'Mean magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "mean_dr3": ( + "Mean magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 'std_dr3': ( - 'Magnitude standard deviation after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "std_dr3": ( + "Magnitude standard deviation after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 'f_dr3': ( - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' - ' (n_dr3/n_mag_est_ok)' + "f_dr3": ( + "Fraction of mag-est-ok samples with centroid residual < 3 arcsec" + " (n_dr3/n_mag_est_ok)" ), - 'n_dr3': 'Number of mag-est-ok samples with centroid residual < 3 arcsec', - 'n_dr3_outliers': ( - 'Number of magnitude outliers after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "n_dr3": "Number of mag-est-ok samples with centroid residual < 3 arcsec", + "n_dr3_outliers": ( + "Number of magnitude outliers after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 'median_dr3': ( - 'Median magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "median_dr3": ( + "Median magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 'sigma_minus_dr3': ( - '15.8% quantile of magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "sigma_minus_dr3": ( + "15.8% quantile of magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 'sigma_plus_dr3': ( - '84.2% quantile of magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis' + "sigma_plus_dr3": ( + "84.2% quantile of magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" ), - 't_mean_dbox5': ( - 'Truncated mean magnitude after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis' + "t_mean_dbox5": ( + "Truncated mean magnitude after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" ), - 't_std_dbox5': ( - 'Truncated magnitude standard deviation after removing outliers and samples' - ' with centroid residual out of a 5 arcsec box, on a per-observation basis' + "t_std_dbox5": ( + "Truncated magnitude standard deviation after removing outliers and samples" + " with centroid residual out of a 5 arcsec box, on a per-observation basis" ), - 'mean_dbox5': ( - 'Mean magnitude after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis' + "mean_dbox5": ( + "Mean magnitude after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" ), - 'std_dbox5': ( - 'Magnitude standard deviation after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis' + "std_dbox5": ( + "Magnitude standard deviation after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" ), - 'f_dbox5': ( - 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box' + "f_dbox5": ( + "Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box" ), - 'n_dbox5': ( - 'Number of mag-est-ok samples with centroid residual within a 5 arcsec box' + "n_dbox5": ( + "Number of mag-est-ok samples with centroid residual within a 5 arcsec box" ), - 'n_dbox5_outliers': ( - 'Number of magnitude outliers after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis' + "n_dbox5_outliers": ( + "Number of magnitude outliers after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" ), - 'median_dbox5': ( - 'Median magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis' + "median_dbox5": ( + "Median magnitude after removing outliers and samples with " + "angular residual out of a 5 arcsec box, on a per-observation basis" ), - 'sigma_minus_dbox5': ( - '15.8% quantile of magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis' + "sigma_minus_dbox5": ( + "15.8% quantile of magnitude after removing outliers and samples with " + "angular residual out of a 5 arcsec box, on a per-observation basis" ), - 'sigma_plus_dbox5': ( - '84.2% quantile of magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis' + "sigma_plus_dbox5": ( + "84.2% quantile of magnitude after removing outliers and samples with " + "angular residual out of a 5 arcsec box, on a per-observation basis" ), } @@ -1141,7 +1141,7 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): :return: dict dictionary with stats """ - logger.debug(f'Getting stats for AGASC ID {agasc_id}...') + logger.debug(f"Getting stats for AGASC ID {agasc_id}...") min_mag_obs_err = 0.03 if not obs_status_override: obs_status_override = {} @@ -1149,77 +1149,77 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): star_obs_catalogs.load(tstop=tstop) # Get a table of every time the star has been observed star_obs = star_obs_catalogs.STARS_OBS[ - star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id + star_obs_catalogs.STARS_OBS["agasc_id"] == agasc_id ] if len(star_obs) > 1: - star_obs = star_obs.loc['mp_starcat_time', sorted(star_obs['mp_starcat_time'])] + star_obs = star_obs.loc["mp_starcat_time", sorted(star_obs["mp_starcat_time"])] # this is the default result, if nothing gets calculated result = { - 'last_obs_time': 0, - 'agasc_id': agasc_id, - 'mag_aca': np.nan, - 'mag_aca_err': np.nan, - 'mag_obs': 0.0, - 'mag_obs_err': np.nan, - 'mag_obs_std': 0.0, - 'color': np.nan, - 'n_obsids': 0, - 'n_obsids_fail': 0, - 'n_obsids_suspect': 0, - 'n_obsids_ok': 0, - 'n_no_mag': 0, - 'n': 0, - 'n_ok': 0, - 'n_ok_3': 0, - 'n_ok_5': 0, - 'n_mag_est_ok': 0, - 'n_mag_est_ok_3': 0, - 'n_mag_est_ok_5': 0, - 'f_ok': 0.0, - 'median': 0, - 'sigma_minus': 0, - 'sigma_plus': 0, - 'mean': 0, - 'std': 0, - 'mag_weighted_mean': 0, - 'mag_weighted_std': 0, - 't_mean': 0, - 't_std': 0, - 'n_outlier': 0, - 't_mean_1': 0, - 't_std_1': 0, - 'n_outlier_1': 0, - 't_mean_2': 0, - 't_std_2': 0, - 'n_outlier_2': 0, + "last_obs_time": 0, + "agasc_id": agasc_id, + "mag_aca": np.nan, + "mag_aca_err": np.nan, + "mag_obs": 0.0, + "mag_obs_err": np.nan, + "mag_obs_std": 0.0, + "color": np.nan, + "n_obsids": 0, + "n_obsids_fail": 0, + "n_obsids_suspect": 0, + "n_obsids_ok": 0, + "n_no_mag": 0, + "n": 0, + "n_ok": 0, + "n_ok_3": 0, + "n_ok_5": 0, + "n_mag_est_ok": 0, + "n_mag_est_ok_3": 0, + "n_mag_est_ok_5": 0, + "f_ok": 0.0, + "median": 0, + "sigma_minus": 0, + "sigma_plus": 0, + "mean": 0, + "std": 0, + "mag_weighted_mean": 0, + "mag_weighted_std": 0, + "t_mean": 0, + "t_std": 0, + "n_outlier": 0, + "t_mean_1": 0, + "t_std_1": 0, + "n_outlier_1": 0, + "t_mean_2": 0, + "t_std_2": 0, + "n_outlier_2": 0, # these are the criteria for including in supplement - 'selected_atol': False, - 'selected_rtol': False, - 'selected_mag_aca_err': False, - 'selected_color': False, - 'f_mag_est_ok': 0, - 'f_mag_est_ok_3': 0, - 'f_mag_est_ok_5': 0, - 'f_ok_3': 0, - 'f_ok_5': 0, + "selected_atol": False, + "selected_rtol": False, + "selected_mag_aca_err": False, + "selected_color": False, + "f_mag_est_ok": 0, + "f_mag_est_ok_3": 0, + "f_mag_est_ok_5": 0, + "f_ok_3": 0, + "f_ok_5": 0, } - for tag in ['dr3', 'dbox5']: + for tag in ["dr3", "dbox5"]: result.update( { - f't_mean_{tag}': 0, - f't_std_{tag}': 0, - f't_mean_{tag}_not': 0, - f't_std_{tag}_not': 0, - f'mean_{tag}': 0, - f'std_{tag}': 0, - f'f_{tag}': 0, - f'n_{tag}': 0, - f'n_{tag}_outliers': 0, - f'median_{tag}': 0, - f'sigma_minus_{tag}': 0, - f'sigma_plus_{tag}': 0, + f"t_mean_{tag}": 0, + f"t_std_{tag}": 0, + f"t_mean_{tag}_not": 0, + f"t_std_{tag}_not": 0, + f"mean_{tag}": 0, + f"std_{tag}": 0, + f"f_{tag}": 0, + f"n_{tag}": 0, + f"n_{tag}_outliers": 0, + f"median_{tag}": 0, + f"sigma_minus_{tag}": 0, + f"sigma_plus_{tag}": 0, } ) @@ -1230,14 +1230,14 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): [ ( (oi, ai) in obs_status_override - and obs_status_override[(oi, ai)]['status'] != 0 + and obs_status_override[(oi, ai)]["status"] != 0 ) - for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']] + for oi, ai in star_obs[["mp_starcat_time", "agasc_id"]] ] ) if np.any(excluded_obs): logger.debug( - ' Excluding observations flagged in obs-status table: ' + " Excluding observations flagged in obs-status table: " f'{list(star_obs[excluded_obs]["obsid"])}' ) @@ -1245,14 +1245,14 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): [ ( (oi, ai) in obs_status_override - and obs_status_override[(oi, ai)]['status'] == 0 + and obs_status_override[(oi, ai)]["status"] == 0 ) - for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']] + for oi, ai in star_obs[["mp_starcat_time", "agasc_id"]] ] ) if np.any(included_obs): logger.debug( - ' Including observations marked OK in obs-status table: ' + " Including observations marked OK in obs-status table: " f'{list(star_obs[included_obs]["obsid"])}' ) @@ -1261,43 +1261,44 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): stats = [] last_obs_time = 0 for i, obs in enumerate(star_obs): - oi, ai = obs['mp_starcat_time', 'agasc_id'] - comment = '' + oi, ai = obs["mp_starcat_time", "agasc_id"] + comment = "" if (oi, ai) in obs_status_override: status = obs_status_override[(oi, ai)] logger.debug( - f' overriding status for (AGASC ID {ai}, starcat time {oi}): ' + f" overriding status for (AGASC ID {ai}, starcat time {oi}): " f'{status["status"]}, {status["comments"]}' ) - comment = status['comments'] + comment = status["comments"] try: - last_obs_time = CxoTime(obs['mp_starcat_time']).cxcsec + last_obs_time = CxoTime(obs["mp_starcat_time"]).cxcsec telem = Table(get_telemetry(obs)) obs_stat = get_obs_stats(obs, telem={k: telem[k] for k in telem.colnames}) obs_stat.update( { - 'obs_ok': included_obs[i] | ( + "obs_ok": included_obs[i] + | ( ~excluded_obs[i] - & (obs_stat['n'] > 10) - & (obs_stat['f_mag_est_ok'] > 0.3) - & (obs_stat['lf_variability_100s'] < 1) + & (obs_stat["n"] > 10) + & (obs_stat["f_mag_est_ok"] > 0.3) + & (obs_stat["lf_variability_100s"] < 1) ), - 'obs_suspect': False, - 'obs_fail': False, - 'comments': comment, + "obs_suspect": False, + "obs_fail": False, + "comments": comment, } ) all_telem.append(telem) stats.append(obs_stat) - if not obs_stat['obs_ok'] and not excluded_obs[i]: - obs_stat['obs_suspect'] = True + if not obs_stat["obs_ok"] and not excluded_obs[i]: + obs_stat["obs_suspect"] = True failures.append( dict( MagStatsException( - msg='Suspect observation', - agasc_id=obs['agasc_id'], - obsid=obs['obsid'], + msg="Suspect observation", + agasc_id=obs["agasc_id"], + obsid=obs["obsid"], mp_starcat_time=obs["mp_starcat_time"], ) ) @@ -1309,74 +1310,74 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): obs_stat = get_obs_stats(obs, telem=[]) obs_stat.update( { - 'obs_ok': False, - 'obs_suspect': False, - 'obs_fail': True, - 'comments': comment if excluded_obs[i] else f'Error: {e.msg}.', + "obs_ok": False, + "obs_suspect": False, + "obs_fail": True, + "comments": comment if excluded_obs[i] else f"Error: {e.msg}.", } ) stats.append(obs_stat) if not excluded_obs[i]: logger.debug( - f' Error in get_agasc_id_stats({agasc_id=},' + f" Error in get_agasc_id_stats({agasc_id=}," f' obsid={obs["obsid"]}): {e}' ) failures.append(dict(e)) stats = Table(stats) - stats['w'] = np.nan - stats['mean_corrected'] = np.nan - stats['weighted_mean'] = np.nan + stats["w"] = np.nan + stats["mean_corrected"] = np.nan + stats["weighted_mean"] = np.nan star = get_star(agasc_id, use_supplement=False) result.update( { - 'last_obs_time': last_obs_time, - 'mag_aca': star['MAG_ACA'], - 'mag_aca_err': star['MAG_ACA_ERR'] / 100, - 'color': star['COLOR1'], - 'n_obsids_fail': len(failures), - 'n_obsids_suspect': np.count_nonzero(stats['obs_suspect']), - 'n_obsids': n_obsids, + "last_obs_time": last_obs_time, + "mag_aca": star["MAG_ACA"], + "mag_aca_err": star["MAG_ACA_ERR"] / 100, + "color": star["COLOR1"], + "n_obsids_fail": len(failures), + "n_obsids_suspect": np.count_nonzero(stats["obs_suspect"]), + "n_obsids": n_obsids, } ) if not np.any(~excluded_obs): # this happens when all observations have been flagged as not OK a priory (obs-status). logger.debug( - f' Skipping star in get_agasc_id_stats({agasc_id=}).' - ' All observations are flagged as not good.' + f" Skipping star in get_agasc_id_stats({agasc_id=})." + " All observations are flagged as not good." ) return result, stats, failures if len(all_telem) - len(failures) <= 0: # and we reach here if some observations were not flagged as bad, but all failed. logger.debug( - f' Error in get_agasc_id_stats({agasc_id=}): There is no OK observation.' + f" Error in get_agasc_id_stats({agasc_id=}): There is no OK observation." ) return result, stats, failures excluded_obs += np.array([t is None for t in all_telem]) - logger.debug(' identifying outlying observations...') + logger.debug(" identifying outlying observations...") for i, (s, t) in enumerate(zip(stats, all_telem)): if excluded_obs[i]: continue - t['obs_ok'] = np.ones_like(t['mag_est_ok'], dtype=bool) * s['obs_ok'] + t["obs_ok"] = np.ones_like(t["mag_est_ok"], dtype=bool) * s["obs_ok"] logger.debug( - ' identifying outlying observations ' + " identifying outlying observations " f'(OBSID={s["obsid"]}, mp_starcat_time={s["mp_starcat_time"]})' ) - t['obs_outlier'] = np.zeros_like(t['mag_est_ok']) - if np.any(t['mag_est_ok']) and s['f_mag_est_ok'] > 0 and s['obs_ok']: - iqr = s['q75'] - s['q25'] - t['obs_outlier'] = ( - t['mag_est_ok'] + t["obs_outlier"] = np.zeros_like(t["mag_est_ok"]) + if np.any(t["mag_est_ok"]) and s["f_mag_est_ok"] > 0 and s["obs_ok"]: + iqr = s["q75"] - s["q25"] + t["obs_outlier"] = ( + t["mag_est_ok"] & (iqr > 0) & ( - (t['mags'] < s['q25'] - 1.5 * iqr) - | (t['mags'] > s['q75'] + 1.5 * iqr) + (t["mags"] < s["q25"] - 1.5 * iqr) + | (t["mags"] > s["q75"] + 1.5 * iqr) ) ) all_telem = vstack( @@ -1384,32 +1385,31 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): ) n_total = len(all_telem) - kalman = (all_telem['AOACASEQ'] == 'KALM') & (all_telem['AOPCADMD'] == 'NPNT') + kalman = (all_telem["AOACASEQ"] == "KALM") & (all_telem["AOPCADMD"] == "NPNT") all_telem = all_telem[kalman] # non-npm/non-kalman are excluded n_kalman = len(all_telem) - mags = all_telem['mags'] - mag_est_ok = all_telem['mag_est_ok'] & all_telem['obs_ok'] - aca_trak = (all_telem['AOACFCT'] == 'TRAK') & all_telem['obs_ok'] - sat_pix = (all_telem['AOACISP'] == 'OK') & all_telem['obs_ok'] - ion_rad = (all_telem['AOACIIR'] == 'OK') & all_telem['obs_ok'] + mags = all_telem["mags"] + mag_est_ok = all_telem["mag_est_ok"] & all_telem["obs_ok"] + aca_trak = (all_telem["AOACFCT"] == "TRAK") & all_telem["obs_ok"] + sat_pix = (all_telem["AOACISP"] == "OK") & all_telem["obs_ok"] + ion_rad = (all_telem["AOACIIR"] == "OK") & all_telem["obs_ok"] f_mag_est_ok = np.count_nonzero(mag_est_ok) / len(mag_est_ok) result.update( { - 'mag_obs_err': min_mag_obs_err, - 'n_obsids_ok': np.count_nonzero(stats['obs_ok']), - 'n_no_mag': np.count_nonzero((~stats['obs_ok'])) + np.count_nonzero( - stats['f_mag_est_ok'][stats['obs_ok']] < 0.3 - ), - 'n': n_total, - 'n_mag_est_ok': np.count_nonzero(mag_est_ok), - 'f_mag_est_ok': f_mag_est_ok, + "mag_obs_err": min_mag_obs_err, + "n_obsids_ok": np.count_nonzero(stats["obs_ok"]), + "n_no_mag": np.count_nonzero((~stats["obs_ok"])) + + np.count_nonzero(stats["f_mag_est_ok"][stats["obs_ok"]] < 0.3), + "n": n_total, + "n_mag_est_ok": np.count_nonzero(mag_est_ok), + "f_mag_est_ok": f_mag_est_ok, } ) - if result['n_mag_est_ok'] < 10: + if result["n_mag_est_ok"] < 10: return result, stats, failures sigma_minus, q25, median, q75, sigma_plus = np.quantile( @@ -1418,56 +1418,56 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): iqr = q75 - q25 outlier_1 = mag_est_ok & ((mags > q75 + 1.5 * iqr) | (mags < q25 - 1.5 * iqr)) outlier_2 = mag_est_ok & ((mags > q75 + 3 * iqr) | (mags < q25 - 3 * iqr)) - outlier = all_telem['obs_outlier'] + outlier = all_telem["obs_outlier"] # combine measurements using a weighted mean - obs_ok = stats['obs_ok'] - min_std = max(0.1, stats[obs_ok]['std'].min()) - stats['w'][obs_ok] = np.where( - stats['std'][obs_ok] != 0, 1.0 / stats['std'][obs_ok], 1.0 / min_std + obs_ok = stats["obs_ok"] + min_std = max(0.1, stats[obs_ok]["std"].min()) + stats["w"][obs_ok] = np.where( + stats["std"][obs_ok] != 0, 1.0 / stats["std"][obs_ok], 1.0 / min_std ) - stats['mean_corrected'][obs_ok] = ( - stats['t_mean'][obs_ok] + stats['mag_correction'][obs_ok] + stats["mean_corrected"][obs_ok] = ( + stats["t_mean"][obs_ok] + stats["mag_correction"][obs_ok] ) - stats['weighted_mean'][obs_ok] = ( - stats['mean_corrected'][obs_ok] * stats['w'][obs_ok] + stats["weighted_mean"][obs_ok] = ( + stats["mean_corrected"][obs_ok] * stats["w"][obs_ok] ) - mag_weighted_mean = stats[obs_ok]['weighted_mean'].sum() / stats[obs_ok]['w'].sum() + mag_weighted_mean = stats[obs_ok]["weighted_mean"].sum() / stats[obs_ok]["w"].sum() mag_weighted_std = np.sqrt( - ((stats[obs_ok]['mean'] - mag_weighted_mean) ** 2 * stats[obs_ok]['w']).sum() - / stats[obs_ok]['w'].sum() + ((stats[obs_ok]["mean"] - mag_weighted_mean) ** 2 * stats[obs_ok]["w"]).sum() + / stats[obs_ok]["w"].sum() ) result.update( { - 'agasc_id': agasc_id, - 'n_mag_est_ok': np.count_nonzero(mag_est_ok), - 'f_mag_est_ok': f_mag_est_ok, - 'median': median, - 'sigma_minus': sigma_minus, - 'sigma_plus': sigma_plus, - 'mean': np.mean(mags[mag_est_ok]), - 'std': np.std(mags[mag_est_ok]), - 'mag_weighted_mean': mag_weighted_mean, - 'mag_weighted_std': mag_weighted_std, - 't_mean': np.mean(mags[mag_est_ok & (~outlier)]), - 't_std': np.std(mags[mag_est_ok & (~outlier)]), - 'n_outlier': np.count_nonzero(mag_est_ok & outlier), - 't_mean_1': np.mean(mags[mag_est_ok & (~outlier_1)]), - 't_std_1': np.std(mags[mag_est_ok & (~outlier_1)]), - 'n_outlier_1': np.count_nonzero(mag_est_ok & outlier_1), - 't_mean_2': np.mean(mags[mag_est_ok & (~outlier_2)]), - 't_std_2': np.std(mags[mag_est_ok & (~outlier_2)]), - 'n_outlier_2': np.count_nonzero(mag_est_ok & outlier_2), + "agasc_id": agasc_id, + "n_mag_est_ok": np.count_nonzero(mag_est_ok), + "f_mag_est_ok": f_mag_est_ok, + "median": median, + "sigma_minus": sigma_minus, + "sigma_plus": sigma_plus, + "mean": np.mean(mags[mag_est_ok]), + "std": np.std(mags[mag_est_ok]), + "mag_weighted_mean": mag_weighted_mean, + "mag_weighted_std": mag_weighted_std, + "t_mean": np.mean(mags[mag_est_ok & (~outlier)]), + "t_std": np.std(mags[mag_est_ok & (~outlier)]), + "n_outlier": np.count_nonzero(mag_est_ok & outlier), + "t_mean_1": np.mean(mags[mag_est_ok & (~outlier_1)]), + "t_std_1": np.std(mags[mag_est_ok & (~outlier_1)]), + "n_outlier_1": np.count_nonzero(mag_est_ok & outlier_1), + "t_mean_2": np.mean(mags[mag_est_ok & (~outlier_2)]), + "t_std_2": np.std(mags[mag_est_ok & (~outlier_2)]), + "n_outlier_2": np.count_nonzero(mag_est_ok & outlier_2), } ) residual_ok = { - 3: all_telem['dr'] < 3, - 5: (np.abs(all_telem['dy']) < 5) & (np.abs(all_telem['dz']) < 5), + 3: all_telem["dr"] < 3, + 5: (np.abs(all_telem["dy"]) < 5) & (np.abs(all_telem["dz"]) < 5), } - dr_tag = {3: 'dr3', 5: 'dbox5'} + dr_tag = {3: "dr3", 5: "dbox5"} for dr in [3, 5]: tag = dr_tag[dr] k = mag_est_ok & residual_ok[dr] @@ -1478,7 +1478,7 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): sigma_minus, q25, median, q75, sigma_plus = np.quantile( mags[k], [0.158, 0.25, 0.5, 0.75, 0.842] ) - outlier = mag_est_ok & all_telem['obs_outlier'] + outlier = mag_est_ok & all_telem["obs_outlier"] mag_not = ( np.nanmean(mags[k2 & (~outlier)]) if np.count_nonzero(k2 & (~outlier)) @@ -1491,47 +1491,46 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): ) result.update( { - f't_mean_{tag}': np.mean(mags[k & (~outlier)]), - f't_std_{tag}': np.std(mags[k & (~outlier)]), - f't_mean_{tag}_not': mag_not, - f't_std_{tag}_not': std_not, - f'mean_{tag}': np.mean(mags[k]), - f'std_{tag}': np.std(mags[k]), - f'f_{tag}': np.count_nonzero(k) / np.count_nonzero(mag_est_ok), - f'n_{tag}': np.count_nonzero(k), - f'n_{tag}_outliers': np.count_nonzero(k & outlier), - f'f_mag_est_ok_{dr}': np.count_nonzero(k) / len(k), - f'n_mag_est_ok_{dr}': np.count_nonzero(k), - f'median_{tag}': median, - f'sigma_minus_{tag}': sigma_minus, - f'sigma_plus_{tag}': sigma_plus, - f'f_ok_{dr}': (n_ok / n_kalman) if n_kalman else 0, - f'n_ok_{dr}': n_ok, + f"t_mean_{tag}": np.mean(mags[k & (~outlier)]), + f"t_std_{tag}": np.std(mags[k & (~outlier)]), + f"t_mean_{tag}_not": mag_not, + f"t_std_{tag}_not": std_not, + f"mean_{tag}": np.mean(mags[k]), + f"std_{tag}": np.std(mags[k]), + f"f_{tag}": np.count_nonzero(k) / np.count_nonzero(mag_est_ok), + f"n_{tag}": np.count_nonzero(k), + f"n_{tag}_outliers": np.count_nonzero(k & outlier), + f"f_mag_est_ok_{dr}": np.count_nonzero(k) / len(k), + f"n_mag_est_ok_{dr}": np.count_nonzero(k), + f"median_{tag}": median, + f"sigma_minus_{tag}": sigma_minus, + f"sigma_plus_{tag}": sigma_plus, + f"f_ok_{dr}": (n_ok / n_kalman) if n_kalman else 0, + f"n_ok_{dr}": n_ok, } ) result.update( { - 'mag_obs': result['t_mean_dbox5'], - 'mag_obs_err': np.sqrt(result['t_std_dbox5'] ** 2 + min_mag_obs_err**2), - 'mag_obs_std': result['t_std_dbox5'], - 'n_ok': result['n_ok_5'], - 'f_ok': result['f_ok_5'], + "mag_obs": result["t_mean_dbox5"], + "mag_obs_err": np.sqrt(result["t_std_dbox5"] ** 2 + min_mag_obs_err**2), + "mag_obs_std": result["t_std_dbox5"], + "n_ok": result["n_ok_5"], + "f_ok": result["f_ok_5"], } ) # these are the criteria for including in supplement result.update( { - 'selected_atol': np.abs(result['mag_obs'] - result['mag_aca']) > 0.3, - 'selected_rtol': ( - np.abs(result['mag_obs'] - result['mag_aca']) - > 3 * result['mag_aca_err'] - ), - 'selected_mag_aca_err': result['mag_aca_err'] > 0.2, - 'selected_color': (result['color'] == 1.5) | ( - np.isclose(result['color'], 0.7) + "selected_atol": np.abs(result["mag_obs"] - result["mag_aca"]) > 0.3, + "selected_rtol": ( + np.abs(result["mag_obs"] - result["mag_aca"]) + > 3 * result["mag_aca_err"] ), + "selected_mag_aca_err": result["mag_aca_err"] > 0.2, + "selected_color": (result["color"] == 1.5) + | (np.isclose(result["color"], 0.7)), } ) diff --git a/agasc/supplement/magnitudes/mag_estimate_report.py b/agasc/supplement/magnitudes/mag_estimate_report.py index 2a474da7..c4e88bf4 100644 --- a/agasc/supplement/magnitudes/mag_estimate_report.py +++ b/agasc/supplement/magnitudes/mag_estimate_report.py @@ -22,11 +22,11 @@ JINJA2 = jinja2.Environment( - loader=jinja2.PackageLoader('agasc.supplement.magnitudes', 'templates'), - autoescape=jinja2.select_autoescape(['html', 'xml']), + loader=jinja2.PackageLoader("agasc.supplement.magnitudes", "templates"), + autoescape=jinja2.select_autoescape(["html", "xml"]), ) -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") class TableEncoder(json.JSONEncoder): @@ -59,7 +59,7 @@ def default(self, obj): if isinstance(obj, CxoTime): return obj.isot if isinstance(obj, np.ma.core.MaskedArray): - return {'columns': obj.dtype.names, 'data': obj.tolist()} + return {"columns": obj.dtype.names, "data": obj.tolist()} if np.isscalar(obj): return int(obj) if np.isreal(obj): @@ -70,9 +70,9 @@ def default(self, obj): class MagEstimateReport: def __init__( self, - agasc_stats='mag_stats_agasc.fits', - obs_stats='mag_stats_obsid.fits', - directory='./mag_estimates_reports', + agasc_stats="mag_stats_agasc.fits", + obs_stats="mag_stats_obsid.fits", + directory="./mag_estimates_reports", ): if type(agasc_stats) is table.Table: self.agasc_stats = agasc_stats @@ -100,69 +100,69 @@ def single_star_html( self, agasc_id, directory, - static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', - highlight_obs=lambda o: ~o['obs_ok'], + static_dir="https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources", + highlight_obs=lambda o: ~o["obs_ok"], ): - if np.sum(self.agasc_stats['agasc_id'] == agasc_id) == 0: + if np.sum(self.agasc_stats["agasc_id"] == agasc_id) == 0: return - star_template = JINJA2.get_template('star_report.html') + star_template = JINJA2.get_template("star_report.html") directory = Path(directory) if not directory.exists(): - logger.debug(f'making report directory {directory}') + logger.debug(f"making report directory {directory}") directory.mkdir(parents=True) - obs_stat = self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + obs_stat = self.obs_stats[self.obs_stats["agasc_id"] == agasc_id] if len(obs_stat) == 0: - raise Exception(f'agasc_id {agasc_id} has not observations') - obs_stat.sort(keys=['mp_starcat_time']) - agasc_stat = dict(self.agasc_stats[self.agasc_stats['agasc_id'] == agasc_id][0]) - agasc_stat['n_obs_bad'] = agasc_stat['n_obsids'] - agasc_stat['n_obsids_ok'] - agasc_stat['last_obs'] = ':'.join( - obs_stat[-1]['mp_starcat_time'].split(':')[:4] + raise Exception(f"agasc_id {agasc_id} has not observations") + obs_stat.sort(keys=["mp_starcat_time"]) + agasc_stat = dict(self.agasc_stats[self.agasc_stats["agasc_id"] == agasc_id][0]) + agasc_stat["n_obs_bad"] = agasc_stat["n_obsids"] - agasc_stat["n_obsids_ok"] + agasc_stat["last_obs"] = ":".join( + obs_stat[-1]["mp_starcat_time"].split(":")[:4] ) # OBSIDs can be repeated - obsids = list(np.unique(obs_stat[highlight_obs(obs_stat)]['obsid'])) + obsids = list(np.unique(obs_stat[highlight_obs(obs_stat)]["obsid"])) args = [ { - 'only_ok': False, - 'draw_agasc_mag': True, - 'draw_legend': True, - 'ylim': 'max', + "only_ok": False, + "draw_agasc_mag": True, + "draw_legend": True, + "ylim": "max", }, { - 'title': 'Magnitude Estimates', - 'only_ok': True, - 'ylim': 'stats', - 'highlight_obsid': obsids, - 'draw_obs_mag_stats': True, - 'draw_agasc_mag_stats': True, - 'draw_legend': True, - 'outside_markers': True, + "title": "Magnitude Estimates", + "only_ok": True, + "ylim": "stats", + "highlight_obsid": obsids, + "draw_obs_mag_stats": True, + "draw_agasc_mag_stats": True, + "draw_legend": True, + "outside_markers": True, }, - {'type': 'flags'}, + {"type": "flags"}, ] for obsid in obsids: args.append( { - 'obsid': obsid, - 'ylim': 'fit', - 'only_ok': False, - 'draw_obs_mag_stats': True, - 'draw_agasc_mag_stats': True, - 'draw_legend': True, - 'draw_roll_mean': True, - 'outside_markers': True, + "obsid": obsid, + "ylim": "fit", + "only_ok": False, + "draw_obs_mag_stats": True, + "draw_agasc_mag_stats": True, + "draw_legend": True, + "draw_roll_mean": True, + "outside_markers": True, } ) - args.append({'type': 'flags', 'obsid': obsid}) - fig = self.plot_set(agasc_id, args=args, filename=directory / 'mag_stats.png') + args.append({"type": "flags", "obsid": obsid}) + fig = self.plot_set(agasc_id, args=args, filename=directory / "mag_stats.png") plt.close(fig) - with open(directory / 'index.html', 'w') as out: + with open(directory / "index.html", "w") as out: with warnings.catch_warnings(): warnings.filterwarnings( "ignore", @@ -176,18 +176,18 @@ def single_star_html( glossary=GLOSSARY, ) ) - with open(directory / 'data.json', 'w') as json_out: + with open(directory / "data.json", "w") as json_out: json.dump( { - 'agasc_stats': agasc_stat, - 'obs_stats': obs_stat, - 'static_dir': static_dir, - 'glossary': GLOSSARY, + "agasc_stats": agasc_stat, + "obs_stats": obs_stat, + "static_dir": static_dir, + "glossary": GLOSSARY, }, json_out, cls=TableEncoder, ) - return directory / 'index.html' + return directory / "index.html" def multi_star_html( self, @@ -201,8 +201,8 @@ def multi_star_html( include_all_stars=False, make_single_reports=True, nav_links=None, - highlight_obs=lambda o: ~o['obs_ok'], - static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', + highlight_obs=lambda o: ~o["obs_ok"], + static_dir="https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources", no_progress=None, ): if sections is None: @@ -211,10 +211,10 @@ def multi_star_html( # Copying this because it is a dict that will get modified sections = copy.deepcopy(sections) - run_template = JINJA2.get_template('run_report.html') + run_template = JINJA2.get_template("run_report.html") updated_star_ids = ( - updated_stars['agasc_id'] + updated_stars["agasc_id"] if updated_stars is not None and len(updated_stars) else [] ) @@ -222,17 +222,17 @@ def multi_star_html( updated_stars = [] info = { - 'tstart': ( + "tstart": ( tstart if tstart - else CxoTime(self.obs_stats['mp_starcat_time']).min().date + else CxoTime(self.obs_stats["mp_starcat_time"]).min().date ), - 'tstop': ( + "tstop": ( tstop if tstop - else CxoTime(self.obs_stats['mp_starcat_time']).max().date + else CxoTime(self.obs_stats["mp_starcat_time"]).max().date ), - 'report_date': report_date if report_date else CxoTime.now().date, + "report_date": report_date if report_date else CxoTime.now().date, } if filename is None: @@ -241,31 +241,31 @@ def multi_star_html( # this is the list of agasc_id for which we will generate individual reports (if possible) if sections: agasc_ids = np.concatenate( - [np.array(s['stars'], dtype=int) for s in sections] + [np.array(s["stars"], dtype=int) for s in sections] ) else: agasc_ids = [] if include_all_stars: sections.append( { - 'id': 'other_stars', - 'title': 'Other Stars', - 'stars': self.agasc_stats['agasc_id'][ - ~np.in1d(self.agasc_stats['agasc_id'], agasc_ids) + "id": "other_stars", + "title": "Other Stars", + "stars": self.agasc_stats["agasc_id"][ + ~np.in1d(self.agasc_stats["agasc_id"], agasc_ids) ], } ) - agasc_ids = self.agasc_stats['agasc_id'] + agasc_ids = self.agasc_stats["agasc_id"] failed_agasc_ids = [ - f['agasc_id'] + f["agasc_id"] for f in fails - if f['agasc_id'] and int(f['agasc_id']) in self.obs_stats['agasc_id'] + if f["agasc_id"] and int(f["agasc_id"]) in self.obs_stats["agasc_id"] ] agasc_ids = np.unique(np.concatenate([agasc_ids, failed_agasc_ids])) # this turns all None into '' in a new list of failures fails = [ - {k: '' if v is None else v for k, v in f.items()} + {k: "" if v is None else v for k, v in f.items()} for i, f in enumerate(fails) ] @@ -273,59 +273,59 @@ def multi_star_html( # add some extra fields if len(agasc_stats): - agasc_stats['n_obs_bad_fail'] = agasc_stats['n_obsids_fail'] - agasc_stats['n_obs_bad'] = ( - agasc_stats['n_obsids'] - agasc_stats['n_obsids_ok'] + agasc_stats["n_obs_bad_fail"] = agasc_stats["n_obsids_fail"] + agasc_stats["n_obs_bad"] = ( + agasc_stats["n_obsids"] - agasc_stats["n_obsids_ok"] ) - agasc_stats['flag'] = ' ' - agasc_stats['flag'][:] = '' - agasc_stats['flag'][ - (agasc_stats['n_obs_bad'] > 0) | (agasc_stats['n_obsids'] == 0) - ] = 'warning' - agasc_stats['flag'][agasc_stats['n_obs_bad_fail'] > 0] = 'danger' - agasc_stats['delta'] = agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'] - agasc_stats['sigma'] = ( - agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca'] - ) / agasc_stats['mag_aca_err'] - agasc_stats['new'] = True - agasc_stats['new'][ - np.in1d(agasc_stats['agasc_id'], updated_star_ids) + agasc_stats["flag"] = " " + agasc_stats["flag"][:] = "" + agasc_stats["flag"][ + (agasc_stats["n_obs_bad"] > 0) | (agasc_stats["n_obsids"] == 0) + ] = "warning" + agasc_stats["flag"][agasc_stats["n_obs_bad_fail"] > 0] = "danger" + agasc_stats["delta"] = agasc_stats["t_mean_dr3"] - agasc_stats["mag_aca"] + agasc_stats["sigma"] = ( + agasc_stats["t_mean_dr3"] - agasc_stats["mag_aca"] + ) / agasc_stats["mag_aca_err"] + agasc_stats["new"] = True + agasc_stats["new"][ + np.in1d(agasc_stats["agasc_id"], updated_star_ids) ] = False - agasc_stats['update_mag_aca'] = np.nan - agasc_stats['update_mag_aca_err'] = np.nan - agasc_stats['last_obs'] = CxoTime(agasc_stats['last_obs_time']).date + agasc_stats["update_mag_aca"] = np.nan + agasc_stats["update_mag_aca_err"] = np.nan + agasc_stats["last_obs"] = CxoTime(agasc_stats["last_obs_time"]).date if len(updated_stars): - agasc_stats['update_mag_aca'][ - np.in1d(agasc_stats['agasc_id'], updated_star_ids) - ] = updated_stars['mag_aca'] - agasc_stats['update_mag_aca_err'][ - np.in1d(agasc_stats['agasc_id'], updated_star_ids) - ] = updated_stars['mag_aca_err'] + agasc_stats["update_mag_aca"][ + np.in1d(agasc_stats["agasc_id"], updated_star_ids) + ] = updated_stars["mag_aca"] + agasc_stats["update_mag_aca_err"][ + np.in1d(agasc_stats["agasc_id"], updated_star_ids) + ] = updated_stars["mag_aca_err"] tooltips = { - 'warning': 'At least one bad observation', - 'danger': 'At least failed observation', + "warning": "At least one bad observation", + "danger": "At least failed observation", } # make all individual star reports star_reports = {} - logger.debug('-' * 80) + logger.debug("-" * 80) logger.info("Generating star reports") for agasc_id in tqdm( np.atleast_1d(agasc_ids).tolist(), - desc='progress', + desc="progress", disable=no_progress, - unit='star', + unit="star", ): try: - logger.debug('-' * 80) - logger.debug(f'{agasc_id=}') + logger.debug("-" * 80) + logger.debug(f"{agasc_id=}") dirname = ( self.directory - / 'stars' - / f'{agasc_id//1e7:03.0f}' - / f'{agasc_id:.0f}' + / "stars" + / f"{agasc_id//1e7:03.0f}" + / f"{agasc_id:.0f}" ) if make_single_reports: self.single_star_html( @@ -333,14 +333,14 @@ def multi_star_html( ) star_reports[agasc_id] = dirname except mag_estimate.MagStatsException: - logger.debug(f' Error generating report for {agasc_id=}') + logger.debug(f" Error generating report for {agasc_id=}") # remove empty sections, and set the star tables for each of the remaining sections sections = sections.copy() - sections = [section for section in sections if len(section['stars'])] + sections = [section for section in sections if len(section["stars"])] for section in sections: - section['stars'] = agasc_stats[ - np.in1d(agasc_stats['agasc_id'], section['stars']) + section["stars"] = agasc_stats[ + np.in1d(agasc_stats["agasc_id"], section["stars"]) ].as_array() # this is a hack @@ -350,7 +350,7 @@ def multi_star_html( # make report if not self.directory.exists(): self.directory.mkdir(parents=True) - with open(self.directory / filename, 'w') as out: + with open(self.directory / filename, "w") as out: with warnings.catch_warnings(): warnings.filterwarnings( "ignore", @@ -369,19 +369,19 @@ def multi_star_html( ) ) - json_filename = filename.replace('.html', '.json') + json_filename = filename.replace(".html", ".json") if json_filename == filename: - json_filename = filename + '.json' - with open(self.directory / json_filename, 'w') as json_out: + json_filename = filename + ".json" + with open(self.directory / json_filename, "w") as json_out: json.dump( { - 'info': info, - 'sections': sections, - 'failures': fails, - 'star_reports': star_reports, - 'tooltips': tooltips, - 'static_dir': static_dir, - 'glossary': GLOSSARY, + "info": info, + "sections": sections, + "failures": fails, + "star_reports": star_reports, + "tooltips": tooltips, + "static_dir": static_dir, + "glossary": GLOSSARY, }, json_out, cls=TableEncoder, @@ -401,22 +401,22 @@ def plot_agasc_id_single( draw_agasc_mag=False, draw_roll_mean=False, draw_legend=False, - ylim='fit', + ylim="fit", ax=None, outside_markers=False, ): if title is not None: ax.set_title(title) elif obsid is not None: - ax.set_title(f'OBSID {obsid}') + ax.set_title(f"OBSID {obsid}") if type(highlight_obsid) is not list and np.isscalar(highlight_obsid): highlight_obsid = [highlight_obsid] - agasc_stat = self.agasc_stats[self.agasc_stats['agasc_id'] == agasc_id][0] - obs_stats = self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + agasc_stat = self.agasc_stats[self.agasc_stats["agasc_id"] == agasc_id][0] + obs_stats = self.obs_stats[self.obs_stats["agasc_id"] == agasc_id] arg_obsid = obsid if arg_obsid: - obs_stats = obs_stats[obs_stats['obsid'] == arg_obsid] + obs_stats = obs_stats[obs_stats["obsid"] == arg_obsid] previous_axes = plt.gca() if ax is not None: @@ -430,104 +430,104 @@ def plot_agasc_id_single( ) telem = mag_estimate.add_obs_info(telem, obs_stats) except Exception as e: - logger.debug(f'Error making plot: {e}') + logger.debug(f"Error making plot: {e}") telem = [] if len(telem) == 0 or ( - arg_obsid is not None and np.sum(telem['obsid'] == arg_obsid) == 0 + arg_obsid is not None and np.sum(telem["obsid"] == arg_obsid) == 0 ): - msg = 'No Telemetry' + msg = "No Telemetry" if arg_obsid is not None: - msg += f' for OBSID {arg_obsid}' + msg += f" for OBSID {arg_obsid}" ax.text( np.mean(ax.get_xlim()), np.mean(ax.get_ylim()), msg, - horizontalalignment='center', - verticalalignment='center', + horizontalalignment="center", + verticalalignment="center", ) return - obsids = [arg_obsid] if arg_obsid else np.unique(telem['obsid']) + obsids = [arg_obsid] if arg_obsid else np.unique(telem["obsid"]) - timeline = telem[['times', 'mags', 'obsid', 'obs_outlier']].copy() - timeline['index'] = np.arange(len(timeline)) - timeline['mean'] = np.nan - timeline['std'] = np.nan - timeline['mag_mean'] = np.nan - timeline['mag_std'] = np.nan - for i, obsid in enumerate(np.unique(timeline['obsid'])): - sel = obs_stats['obsid'] == obsid + timeline = telem[["times", "mags", "obsid", "obs_outlier"]].copy() + timeline["index"] = np.arange(len(timeline)) + timeline["mean"] = np.nan + timeline["std"] = np.nan + timeline["mag_mean"] = np.nan + timeline["mag_std"] = np.nan + for i, obsid in enumerate(np.unique(timeline["obsid"])): + sel = obs_stats["obsid"] == obsid if draw_obs_mag_stats and np.any(sel): - timeline['mag_mean'][timeline['obsid'] == obsid] = obs_stats[sel][ - 'mean' + timeline["mag_mean"][timeline["obsid"] == obsid] = obs_stats[sel][ + "mean" ][0] - timeline['mag_std'][timeline['obsid'] == obsid] = obs_stats[sel]['std'][ + timeline["mag_std"][timeline["obsid"] == obsid] = obs_stats[sel]["std"][ 0 ] timeline = timeline.as_array() ok = ( - (telem['AOACASEQ'] == 'KALM') - & (telem['AOACIIR'] == 'OK') - & (telem['AOPCADMD'] == 'NPNT') + (telem["AOACASEQ"] == "KALM") + & (telem["AOACIIR"] == "OK") + & (telem["AOPCADMD"] == "NPNT") ) - ok = ok & (telem['dr'] < 5) + ok = ok & (telem["dr"] < 5) # set the limits of the plot beforehand - ok_ylim = ok & (telem['mags'] > 0) + ok_ylim = ok & (telem["mags"] > 0) ylims_set = False if arg_obsid is not None: - ok_ylim = ok_ylim & (telem['obsid'] == arg_obsid) - if ylim == 'fit': + ok_ylim = ok_ylim & (telem["obsid"] == arg_obsid) + if ylim == "fit": if np.sum(ok_ylim): - q25, q50, q75 = np.quantile(telem['mags'][ok_ylim], [0.25, 0.5, 0.75]) + q25, q50, q75 = np.quantile(telem["mags"][ok_ylim], [0.25, 0.5, 0.75]) else: - q25, q50, q75 = np.quantile(telem['mags'], [0.25, 0.5, 0.75]) + q25, q50, q75 = np.quantile(telem["mags"], [0.25, 0.5, 0.75]) iqr = max(q75 - q25, 0.05) ax.set_ylim((q25 - 2 * iqr, q75 + 2 * iqr)) ylims_set = True - elif ylim == 'stats': + elif ylim == "stats": if arg_obsid is not None: - q25, q50, q75 = obs_stats[['q25', 'median', 'q75']][0] + q25, q50, q75 = obs_stats[["q25", "median", "q75"]][0] iqr = max(q75 - q25, 0.05) ax.set_ylim((q25 - 3 * iqr, q75 + 3 * iqr)) ylims_set = True - elif arg_obsid is None and agasc_stat['mag_obs_std'] > 0: + elif arg_obsid is None and agasc_stat["mag_obs_std"] > 0: ylim = ( - agasc_stat['mag_obs'] - 6 * agasc_stat['mag_obs_std'], - agasc_stat['mag_obs'] + 6 * agasc_stat['mag_obs_std'], + agasc_stat["mag_obs"] - 6 * agasc_stat["mag_obs_std"], + agasc_stat["mag_obs"] + 6 * agasc_stat["mag_obs_std"], ) ax.set_ylim(ylim) ylims_set = True - if ylim == 'max' or not ylims_set: + if ylim == "max" or not ylims_set: if np.any(ok_ylim): - ymin, ymax = np.min(telem['mags'][ok_ylim]), np.max( - telem['mags'][ok_ylim] + ymin, ymax = np.min(telem["mags"][ok_ylim]), np.max( + telem["mags"][ok_ylim] ) else: - ymin, ymax = np.min(telem['mags']), np.max(telem['mags']) + ymin, ymax = np.min(telem["mags"]), np.max(telem["mags"]) dy = max(0.3, ymax - ymin) ax.set_ylim((ymin - 0.1 * dy, ymax + 0.1 * dy)) # set flags for different categories of markers - highlighted = np.zeros(len(timeline['times']), dtype=bool) + highlighted = np.zeros(len(timeline["times"]), dtype=bool) if highlight_obsid: - highlighted = highlighted | np.in1d(timeline['obsid'], highlight_obsid) + highlighted = highlighted | np.in1d(timeline["obsid"], highlight_obsid) if highlight_outliers: - highlighted = highlighted | timeline['obs_outlier'] + highlighted = highlighted | timeline["obs_outlier"] ymin, ymax = ax.get_ylim() ymin, ymax = (ymin + 0.01 * (ymax - ymin)), (ymax - 0.01 * (ymax - ymin)) - top = np.ones_like(timeline['mags']) * ymax - bottom = np.ones_like(timeline['mags']) * ymin + top = np.ones_like(timeline["mags"]) * ymax + bottom = np.ones_like(timeline["mags"]) * ymin if outside_markers: - outside_up = timeline['mags'] >= ymax - outside_down = timeline['mags'] <= ymin + outside_up = timeline["mags"] >= ymax + outside_down = timeline["mags"] <= ymin else: - outside_up = np.zeros_like(timeline['mags'], dtype=bool) - outside_down = np.zeros_like(timeline['mags'], dtype=bool) + outside_up = np.zeros_like(timeline["mags"], dtype=bool) + outside_down = np.zeros_like(timeline["mags"], dtype=bool) inside = ~outside_up & ~outside_down # loop over obsids, making each plot @@ -535,108 +535,108 @@ def plot_agasc_id_single( line_handles = [] limits = {} for i, obsid in enumerate(obsids): - in_obsid = timeline['obsid'] == obsid - if len(timeline[timeline['obsid'] == obsid]) == 0: + in_obsid = timeline["obsid"] == obsid + if len(timeline[timeline["obsid"] == obsid]) == 0: continue limits[obsid] = ( - timeline['index'][timeline['obsid'] == obsid].min(), - timeline['index'][timeline['obsid'] == obsid].max(), + timeline["index"][timeline["obsid"] == obsid].min(), + timeline["index"][timeline["obsid"] == obsid].max(), ) if not only_ok and np.any(in_obsid & ~ok): s = plt.scatter( - timeline['index'][in_obsid & ~ok & inside], - timeline[in_obsid & ~ok & inside]['mags'], + timeline["index"][in_obsid & ~ok & inside], + timeline[in_obsid & ~ok & inside]["mags"], s=10, - marker='.', - color='r', - label='not OK', + marker=".", + color="r", + label="not OK", ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][in_obsid & ~ok & outside_down], + timeline["index"][in_obsid & ~ok & outside_down], bottom[in_obsid & ~ok & outside_down], s=10, - marker='v', - color='r', + marker="v", + color="r", ) _ = plt.scatter( - timeline['index'][in_obsid & ~ok & outside_up], + timeline["index"][in_obsid & ~ok & outside_up], top[in_obsid & ~ok & outside_up], s=10, - marker='^', - color='r', + marker="^", + color="r", ) if np.any(in_obsid & ok & highlighted): s = plt.scatter( - timeline['index'][in_obsid & ok & highlighted & inside], - timeline[in_obsid & ok & highlighted & inside]['mags'], + timeline["index"][in_obsid & ok & highlighted & inside], + timeline[in_obsid & ok & highlighted & inside]["mags"], s=10, - marker='.', - color='orange', - label='Highlighted', + marker=".", + color="orange", + label="Highlighted", ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][in_obsid & ok & highlighted & outside_up], + timeline["index"][in_obsid & ok & highlighted & outside_up], top[in_obsid & ok & highlighted & outside_up], s=10, - marker='^', - color='orange', - label='Highlighted', + marker="^", + color="orange", + label="Highlighted", ) _ = plt.scatter( - timeline['index'][in_obsid & ok & highlighted & outside_down], + timeline["index"][in_obsid & ok & highlighted & outside_down], bottom[in_obsid & ok & highlighted & outside_down], s=10, - marker='v', - color='orange', - label='Highlighted', + marker="v", + color="orange", + label="Highlighted", ) if np.any(in_obsid & ok & ~highlighted): s = plt.scatter( - timeline['index'][in_obsid & ok & ~highlighted & inside], - timeline[in_obsid & ok & ~highlighted & inside]['mags'], + timeline["index"][in_obsid & ok & ~highlighted & inside], + timeline[in_obsid & ok & ~highlighted & inside]["mags"], s=10, - marker='.', - color='k', - label='OK', + marker=".", + color="k", + label="OK", ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][in_obsid & ok & (~highlighted) & outside_down], + timeline["index"][in_obsid & ok & (~highlighted) & outside_down], bottom[in_obsid & ok & ~highlighted & outside_down], s=10, - marker='v', - color='k', - label='OK', + marker="v", + color="k", + label="OK", ) _ = plt.scatter( - timeline['index'][in_obsid & ok & ~highlighted & outside_up], + timeline["index"][in_obsid & ok & ~highlighted & outside_up], top[in_obsid & ok & ~highlighted & outside_up], s=10, - marker='^', - color='k', - label='OK', + marker="^", + color="k", + label="OK", ) - sel = obs_stats['obsid'] == obsid + sel = obs_stats["obsid"] == obsid if draw_obs_mag_stats and np.sum(sel): - label = '' if i else 'mag$_{OBSID}$' - if np.isfinite(obs_stats[sel][0]['t_mean']) and np.isfinite( - obs_stats[sel][0]['t_std'] + label = "" if i else "mag$_{OBSID}$" + if np.isfinite(obs_stats[sel][0]["t_mean"]) and np.isfinite( + obs_stats[sel][0]["t_std"] ): - mag_mean = obs_stats[sel][0]['t_mean'] - mag_mean_minus = mag_mean - obs_stats[sel][0]['t_std'] - mag_mean_plus = mag_mean + obs_stats[sel][0]['t_std'] + mag_mean = obs_stats[sel][0]["t_mean"] + mag_mean_minus = mag_mean - obs_stats[sel][0]["t_std"] + mag_mean_plus = mag_mean + obs_stats[sel][0]["t_std"] lh = ax.plot( limits[obsid], [mag_mean, mag_mean], linewidth=2, - color='orange', + color="orange", label=label, ) if i == 0: @@ -645,25 +645,25 @@ def plot_agasc_id_single( limits[obsid], [mag_mean_minus, mag_mean_minus], [mag_mean_plus, mag_mean_plus], - color='orange', + color="orange", alpha=0.1, zorder=100, ) else: - (ax.plot([], [], linewidth=2, color='orange', label=label)) + (ax.plot([], [], linewidth=2, color="orange", label=label)) if draw_roll_mean: - o = (timeline['obsid'] == obsid) & ok + o = (timeline["obsid"] == obsid) & ok roll_mean = mag_estimate.rolling_mean( - timeline['times'], timeline['mags'], window=100, selection=o + timeline["times"], timeline["mags"], window=100, selection=o ) lh = ax.plot( - timeline['index'], + timeline["index"], roll_mean, - '--', + "--", linewidth=1, - color='r', - label='rolling mean', + color="r", + label="rolling mean", ) if i == 0: line_handles += lh @@ -671,45 +671,45 @@ def plot_agasc_id_single( sorted_obsids = sorted(limits.keys(), key=lambda lim: limits[lim][1]) for i, obsid in enumerate(sorted_obsids): (tmin, tmax) = limits[obsid] - ax.plot([tmin, tmin], ax.get_ylim(), ':', color='purple', scaley=False) + ax.plot([tmin, tmin], ax.get_ylim(), ":", color="purple", scaley=False) shift = 0.07 * (ax.get_ylim()[1] - ax.get_ylim()[0]) * (1 + i % 3) ax.text( np.mean([tmin, tmax]), ax.get_ylim()[0] + shift, - f'{obsid}', - verticalalignment='top', - horizontalalignment='center', + f"{obsid}", + verticalalignment="top", + horizontalalignment="center", ) if limits: tmax = max([v[1] for v in limits.values()]) - ax.plot([tmax, tmax], ax.get_ylim(), ':', color='purple', scaley=False) + ax.plot([tmax, tmax], ax.get_ylim(), ":", color="purple", scaley=False) xlim = ax.get_xlim() ax.set_xlim(xlim) if draw_agasc_mag: - mag_aca = np.mean(agasc_stat['mag_aca']) + mag_aca = np.mean(agasc_stat["mag_aca"]) line_handles += ax.plot( xlim, [mag_aca, mag_aca], - label='mag$_{AGASC}$', - color='green', + label="mag$_{AGASC}$", + color="green", scalex=False, scaley=False, ) if ( draw_agasc_mag_stats - and np.isfinite(agasc_stat['mag_obs']) - and agasc_stat['mag_obs'] > 0 + and np.isfinite(agasc_stat["mag_obs"]) + and agasc_stat["mag_obs"] > 0 ): - mag_weighted_mean = agasc_stat['mag_obs'] - mag_weighted_std = agasc_stat['mag_obs_std'] + mag_weighted_mean = agasc_stat["mag_obs"] + mag_weighted_std = agasc_stat["mag_obs_std"] line_handles += ax.plot( ax.get_xlim(), [mag_weighted_mean, mag_weighted_mean], - label='mag', - color='r', + label="mag", + color="r", scalex=False, ) ax.fill_between( @@ -722,16 +722,16 @@ def plot_agasc_id_single( mag_weighted_mean + mag_weighted_std, mag_weighted_mean + mag_weighted_std, ], - color='r', + color="r", alpha=0.1, ) if draw_legend: ax.set_xlim((xlim[0], xlim[1] + 0.1 * (xlim[1] - xlim[0]))) if marker_handles: - ax.add_artist(plt.legend(handles=marker_handles, loc='lower right')) + ax.add_artist(plt.legend(handles=marker_handles, loc="lower right")) if line_handles: - plt.legend(handles=line_handles, loc='upper right') + plt.legend(handles=line_handles, loc="upper right") plt.sca(previous_axes) @@ -743,90 +743,90 @@ def plot_flags(telemetry, ax=None, obsid=None): if len(telemetry) > 0: timeline = telemetry[ [ - 'times', - 'mags', - 'obsid', - 'obs_ok', - 'dr', - 'AOACFCT', - 'AOACASEQ', - 'AOACIIR', - 'AOPCADMD', + "times", + "mags", + "obsid", + "obs_ok", + "dr", + "AOACFCT", + "AOACASEQ", + "AOACIIR", + "AOPCADMD", ] ] - timeline['x'] = np.arange(len(timeline)) - timeline['y'] = np.ones(len(timeline)) + timeline["x"] = np.arange(len(timeline)) + timeline["y"] = np.ones(len(timeline)) timeline = timeline.as_array() all_ok = ( - (timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] == 'TRAK') - & (timeline['AOACIIR'] == 'OK') - & (timeline['dr'] < 3) + (timeline["AOACASEQ"] == "KALM") + & (timeline["AOPCADMD"] == "NPNT") + & (timeline["AOACFCT"] == "TRAK") + & (timeline["AOACIIR"] == "OK") + & (timeline["dr"] < 3) ) flags = [ ( - 'dr > 5', + "dr > 5", ( - (timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] == 'TRAK') - & (timeline['dr'] >= 5) + (timeline["AOACASEQ"] == "KALM") + & (timeline["AOPCADMD"] == "NPNT") + & (timeline["AOACFCT"] == "TRAK") + & (timeline["dr"] >= 5) ), ), - ('Ion. rad.', (timeline['AOACIIR'] != 'OK')), + ("Ion. rad.", (timeline["AOACIIR"] != "OK")), ( - 'not track', + "not track", ( - (timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] != 'TRAK') + (timeline["AOACASEQ"] == "KALM") + & (timeline["AOPCADMD"] == "NPNT") + & (timeline["AOACFCT"] != "TRAK") ), ), ( - 'not Kalman', + "not Kalman", ( - (timeline['AOACASEQ'] != 'KALM') - | (timeline['AOPCADMD'] != 'NPNT') + (timeline["AOACASEQ"] != "KALM") + | (timeline["AOPCADMD"] != "NPNT") ), ), ] else: timeline_dtype = np.dtype( - [(n, float) for n in ['times', 'mags', 'obsid', 'obs_ok', 'dr']] + [(n, float) for n in ["times", "mags", "obsid", "obs_ok", "dr"]] + [ (n, int) - for n in ['AOACFCT', 'AOACASEQ', 'AOACIIR', 'AOPCADMD', 'x', 'y'] + for n in ["AOACFCT", "AOACASEQ", "AOACIIR", "AOPCADMD", "x", "y"] ] ) timeline = np.array([], dtype=timeline_dtype) all_ok = np.array([], dtype=bool) flags = [ - ('dr > 5', np.array([], dtype=bool)), - ('Ion. rad.', np.array([], dtype=bool)), - ('not track', np.array([], dtype=bool)), - ('not Kalman', np.array([], dtype=bool)), + ("dr > 5", np.array([], dtype=bool)), + ("Ion. rad.", np.array([], dtype=bool)), + ("not track", np.array([], dtype=bool)), + ("not Kalman", np.array([], dtype=bool)), ] if obsid: for i, (l, o) in enumerate(flags): - flags[i] = (l, o[timeline['obsid'] == obsid]) - all_ok = all_ok[timeline['obsid'] == obsid] - timeline = timeline[timeline['obsid'] == obsid] + flags[i] = (l, o[timeline["obsid"] == obsid]) + all_ok = all_ok[timeline["obsid"] == obsid] + timeline = timeline[timeline["obsid"] == obsid] - obsids = np.unique(timeline['obsid']) + obsids = np.unique(timeline["obsid"]) limits = {} if len(timeline) > 0: if obsid is None: - all_ok = timeline['obs_ok'] & all_ok - flags = [('OBS not OK', ~timeline['obs_ok'])] + flags + all_ok = timeline["obs_ok"] & all_ok + flags = [("OBS not OK", ~timeline["obs_ok"])] + flags for i, obsid in enumerate(obsids): limits[obsid] = ( - timeline['x'][timeline['obsid'] == obsid].min(), - timeline['x'][timeline['obsid'] == obsid].max(), + timeline["x"][timeline["obsid"] == obsid].min(), + timeline["x"][timeline["obsid"] == obsid].max(), ) ok = [f[1] for f in flags] @@ -835,68 +835,68 @@ def plot_flags(telemetry, ax=None, obsid=None): for i in range(len(ok)): ax.scatter( - timeline['x'][ok[i]], - ticks[i] * timeline['y'][ok[i]], + timeline["x"][ok[i]], + ticks[i] * timeline["y"][ok[i]], s=4, - marker='.', - color='k', + marker=".", + color="k", ) ax.yaxis.set_major_locator(FixedLocator(ticks)) ax.set_yticklabels(labels) ax.set_ylim((-1, ticks[-1] + 1)) - ax.grid(True, axis='y', linestyle=':') + ax.grid(True, axis="y", linestyle=":") sorted_obsids = sorted(limits.keys(), key=lambda lim: limits[lim][1]) for i, obsid in enumerate(sorted_obsids): (tmin, tmax) = limits[obsid] - ax.axvline(tmin, linestyle=':', color='purple') + ax.axvline(tmin, linestyle=":", color="purple") if i == len(limits) - 1: - ax.axvline(tmax, linestyle=':', color='purple') + ax.axvline(tmax, linestyle=":", color="purple") divider = make_axes_locatable(ax) - ax_dr = divider.append_axes("bottom", size='25%', pad=0.0, sharex=ax) + ax_dr = divider.append_axes("bottom", size="25%", pad=0.0, sharex=ax) if len(timeline) > 0: - dr = timeline['dr'].copy() + dr = timeline["dr"].copy() dr[dr > 10] = 10 ax_dr.scatter( - timeline['x'][all_ok & (dr < 10)], + timeline["x"][all_ok & (dr < 10)], dr[all_ok & (dr < 10)], s=3, - marker='.', - color='k', + marker=".", + color="k", ) ax_dr.scatter( - timeline['x'][all_ok & (dr >= 10)], + timeline["x"][all_ok & (dr >= 10)], dr[all_ok & (dr >= 10)], s=3, - marker='^', - color='k', + marker="^", + color="k", ) ax_dr.scatter( - timeline['x'][~all_ok & (dr < 10)], + timeline["x"][~all_ok & (dr < 10)], dr[~all_ok & (dr < 10)], s=3, - marker='.', - color='r', + marker=".", + color="r", ) ax_dr.scatter( - timeline['x'][~all_ok & (dr >= 10)], + timeline["x"][~all_ok & (dr >= 10)], dr[~all_ok & (dr >= 10)], s=3, - marker='^', - color='r', + marker="^", + color="r", ) - ax_dr.set_ylabel('dr') + ax_dr.set_ylabel("dr") ax_dr.set_ylim((-0.5, 10.5)) ax_dr.set_yticks([0.0, 2.5, 5, 7.5, 10], minor=True) ax_dr.set_yticks([0.0, 5, 10], minor=False) - ax_dr.grid(True, axis='y', linestyle=':') + ax_dr.grid(True, axis="y", linestyle=":") for i, obsid in enumerate(sorted_obsids): (tmin, tmax) = limits[obsid] - ax_dr.axvline(tmin, linestyle=':', color='purple') + ax_dr.axvline(tmin, linestyle=":", color="purple") if i == len(sorted_obsids) - 1: - ax_dr.axvline(tmax, linestyle=':', color='purple') + ax_dr.axvline(tmax, linestyle=":", color="purple") def plot_set(self, agasc_id, args, telem=None, filename=None): if not args: @@ -908,10 +908,10 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): agasc_id, ignore_exceptions=True ) telem = mag_estimate.add_obs_info( - telem, self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + telem, self.obs_stats[self.obs_stats["agasc_id"] == agasc_id] ) except Exception as e: - logger.debug(f'Error making plot: {e}') + logger.debug(f"Error making plot: {e}") telem = [] if len(telem) == 0: @@ -920,11 +920,11 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): fig, ax = plt.subplots(len(args), 1, figsize=(15, 3.5 * len(args))) if len(args) == 1: ax = [ax] - ax[0].set_title(f'AGASC ID {agasc_id}') + ax[0].set_title(f"AGASC ID {agasc_id}") for i, kwargs in enumerate(args): - if 'type' in kwargs and kwargs['type'] == 'flags': - o = kwargs['obsid'] if 'obsid' in kwargs else None + if "type" in kwargs and kwargs["type"] == "flags": + o = kwargs["obsid"] if "obsid" in kwargs else None self.plot_flags(telem, ax[i], obsid=o) if i: ax[i].set_xlim(ax[i - 1].get_xlim()) @@ -933,7 +933,7 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): plt.tight_layout() if filename is not None: - print(f'saving {filename}') + print(f"saving {filename}") fig.savefig(filename) return fig @@ -943,7 +943,7 @@ def email_bad_obs_report( bad_obs, to, sender=f"{getpass.getuser()}@{platform.uname()[1]}" ): date = CxoTime().date[:14] - message = JINJA2.get_template('email_report.txt') + message = JINJA2.get_template("email_report.txt") msg = MIMEText(message) msg["From"] = sender @@ -957,63 +957,63 @@ def email_bad_obs_report( GLOSSARY = { - 'sample': """ + "sample": """ One sample of ACA image telemetry. This could be a real image readout or it could be a null 4x4 image when the ACA is not tracking. Be aware that the sample period is faster for null 4x4 images (1.025 sec) than for typical 6x6 or 8x8 tracked images (2.05 or 4.1 sec respectively).""", - 'Kalman samples': ( - 'Subset of samples when ACA could be tracking stars (AOPCADMD == NPNT &' - ' AOACASEQ == KALM)' + "Kalman samples": ( + "Subset of samples when ACA could be tracking stars (AOPCADMD == NPNT &" + " AOACASEQ == KALM)" ), - 'dr3': """ + "dr3": """ Subset of tracked Kalman samples with radial centroid residual < 3 arcsec. This corresponds to "high quality" readouts. This effectively includes the track subset (AOACFCT == TRAK) because readouts with no TRAK are assigned an infinite centroid residual.""", - 'dbox5': """Subset of tracked Kalman samples with centroid residual within a 5 arcsec box. + "dbox5": """Subset of tracked Kalman samples with centroid residual within a 5 arcsec box. This corresponds to spatial filter used by the OBC for inclusion in Kalman filter. This effectively includes the track subset (AOACFCT == TRAK) because readouts with no TRAK are assigned an infinite centroid residual.""", - 'track': ( - 'Subset of Kalman samples where the image is being tracked (AOACFCT == TRAK)' + "track": ( + "Subset of Kalman samples where the image is being tracked (AOACFCT == TRAK)" ), - 'sat_pix': 'Subset of Kalman samples with saturated pixel flag OK (AOACISP == OK)', - 'ion_rad': ( - 'Subset of Kalman samples with ionizing radiation flag OK (AOACIIR == OK)' + "sat_pix": "Subset of Kalman samples with saturated pixel flag OK (AOACISP == OK)", + "ion_rad": ( + "Subset of Kalman samples with ionizing radiation flag OK (AOACIIR == OK)" ), - 'mag_est_ok': ( + "mag_est_ok": ( "Subset of Kalman samples that have a magnitude estimate (track & ion_rad)" ), - 'n_total': 'Total number of sample regardless of OBC PCAD status', - 'n': 'Synonym for n_total', - 'n_kalman': 'Number of Kalman samples', - 'n_dr3': 'Number of dr3 samples.', - 'n_dbox5': 'Number of dbox5 samples.', - 'n_track': 'Number of track samples.', - 'n_ok_3': 'Number of (track & sat_pix & ion_rad & dr3) samples', - 'n_ok_5': 'Number of (track & sat_pix & ion_rad & dbox5) samples', - 'n_mag_est_ok': 'Number of (track & ion_rad) samples', - 'n_mag_est_ok_3': 'Number of (track & ion_rad & dr3) samples', - 'n_mag_est_ok_5': 'Number of (track & ion_rad & dbox5) samples', - 'f_dr3': ( - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' - '((mag_est_ok & n_dr3)/n_mag_est_ok)' + "n_total": "Total number of sample regardless of OBC PCAD status", + "n": "Synonym for n_total", + "n_kalman": "Number of Kalman samples", + "n_dr3": "Number of dr3 samples.", + "n_dbox5": "Number of dbox5 samples.", + "n_track": "Number of track samples.", + "n_ok_3": "Number of (track & sat_pix & ion_rad & dr3) samples", + "n_ok_5": "Number of (track & sat_pix & ion_rad & dbox5) samples", + "n_mag_est_ok": "Number of (track & ion_rad) samples", + "n_mag_est_ok_3": "Number of (track & ion_rad & dr3) samples", + "n_mag_est_ok_5": "Number of (track & ion_rad & dbox5) samples", + "f_dr3": ( + "Fraction of mag-est-ok samples with centroid residual < 3 arcsec" + "((mag_est_ok & n_dr3)/n_mag_est_ok)" ), - 'f_dbox5': ( - 'Fraction of mag-est-ok samples with centroid within 5 arcsec box' - '((mag_est_ok & n_dbox5)/n_mag_est_ok)' + "f_dbox5": ( + "Fraction of mag-est-ok samples with centroid within 5 arcsec box" + "((mag_est_ok & n_dbox5)/n_mag_est_ok)" ), - 'f_mag_est_ok': ( + "f_mag_est_ok": ( """n_mag_est_ok_3/n_kalman. This is a measure of the fraction of time during an observation that a magnitude estimate is available.""" ), - 'f_mag_est_ok_3': "n_mag_est_ok_3/n_kalman.", - 'f_mag_est_ok_5': "n_mag_est_ok_5/n_kalman.", - 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5', - 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an + "f_mag_est_ok_3": "n_mag_est_ok_3/n_kalman.", + "f_mag_est_ok_5": "n_mag_est_ok_5/n_kalman.", + "f_ok": "n_ok_5 / n_kalman. Same as f_ok_5", + "f_ok_3": """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", - 'f_ok_5': """ + "f_ok_5": """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all. This includes measurements out to 5 arcsec box halfwidth, so potentially 7 arcsec diff --git a/agasc/supplement/magnitudes/star_obs_catalogs.py b/agasc/supplement/magnitudes/star_obs_catalogs.py index 997e7750..b2808a73 100644 --- a/agasc/supplement/magnitudes/star_obs_catalogs.py +++ b/agasc/supplement/magnitudes/star_obs_catalogs.py @@ -20,35 +20,35 @@ def get_star_observations(start=None, stop=None, obsid=None): This is basically the join of kadi.commands.get_observations and kadi.commands.get_starcats, with some extra information (pixel row/col, magnitude error). """ - join_keys = ['starcat_date', 'obsid'] + join_keys = ["starcat_date", "obsid"] observations = Table(commands.get_observations(start=start, stop=stop, obsid=obsid)) - observations = observations[~observations['starcat_date'].mask] + observations = observations[~observations["starcat_date"].mask] # the following line removes manual commands - observations = observations[observations['source'] != 'CMD_EVT'] + observations = observations[observations["source"] != "CMD_EVT"] catalogs = commands.get_starcats_as_table( start=start, stop=stop, obsid=obsid, unique=True ) - catalogs = catalogs[np.in1d(catalogs['type'], ['BOT', 'GUI'])] + catalogs = catalogs[np.in1d(catalogs["type"], ["BOT", "GUI"])] star_obs = join(observations, catalogs, keys=join_keys) - star_obs.rename_columns(['id', 'starcat_date'], ['agasc_id', 'mp_starcat_time']) - star_obs['row'], star_obs['col'] = yagzag_to_pixels( - star_obs['yang'], star_obs['zang'] + star_obs.rename_columns(["id", "starcat_date"], ["agasc_id", "mp_starcat_time"]) + star_obs["row"], star_obs["col"] = yagzag_to_pixels( + star_obs["yang"], star_obs["zang"] ) # Add mag_aca_err column - filename = os.path.join(os.environ['SKA'], 'data', 'agasc', 'proseco_agasc_1p7.h5') + filename = os.path.join(os.environ["SKA"], "data", "agasc", "proseco_agasc_1p7.h5") with tables.open_file(filename) as h5: - agasc_ids = h5.root.data.col('AGASC_ID') - mag_errs = h5.root.data.col('MAG_ACA_ERR') * 0.01 + agasc_ids = h5.root.data.col("AGASC_ID") + mag_errs = h5.root.data.col("MAG_ACA_ERR") * 0.01 - tt = Table([agasc_ids, mag_errs], names=['agasc_id', 'mag_aca_err']) - star_obs = table.join(star_obs, tt, keys='agasc_id') + tt = Table([agasc_ids, mag_errs], names=["agasc_id", "mag_aca_err"]) + star_obs = table.join(star_obs, tt, keys="agasc_id") - star_obs.add_index(['mp_starcat_time']) + star_obs.add_index(["mp_starcat_time"]) - max_time = events.dwells.all().latest('tstart').stop - star_obs = star_obs[star_obs['obs_start'] <= max_time] + max_time = events.dwells.all().latest("tstart").stop + star_obs = star_obs[star_obs["obs_start"] <= max_time] return star_obs diff --git a/agasc/supplement/magnitudes/update_mag_supplement.py b/agasc/supplement/magnitudes/update_mag_supplement.py index 651d942e..d4319a0c 100755 --- a/agasc/supplement/magnitudes/update_mag_supplement.py +++ b/agasc/supplement/magnitudes/update_mag_supplement.py @@ -26,7 +26,7 @@ from cxotime import CxoTime -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") def level0_archive_time_range(): @@ -38,7 +38,7 @@ def level0_archive_time_range(): import sqlite3 import os - db_file = os.path.expandvars('$SKA/data/mica/archive/aca0/archfiles.db3') + db_file = os.path.expandvars("$SKA/data/mica/archive/aca0/archfiles.db3") with sqlite3.connect(db_file) as connection: cursor = connection.cursor() cursor.execute("select tstop from archfiles order by tstop desc limit 1") @@ -68,12 +68,12 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres fails = [] obs_stats = [] agasc_stats = [] - bar = tqdm(agasc_ids, desc='progress', disable=no_progress, unit='star') + bar = tqdm(agasc_ids, desc="progress", disable=no_progress, unit="star") for agasc_id in agasc_ids: bar.update() try: - logger.debug('-' * 80) - logger.debug(f'{agasc_id=}') + logger.debug("-" * 80) + logger.debug(f"{agasc_id=}") agasc_stat, obs_stat, obs_fail = mag_estimate.get_agasc_id_stats( agasc_id=agasc_id, obs_status_override=obs_status_override, tstop=tstop ) @@ -86,7 +86,7 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres fails.append(dict(e)) except Exception as e: # transform Exception to MagStatsException for standard book keeping - msg = f'Unexpected Error: {e}' + msg = f"Unexpected Error: {e}" logger.debug(msg) fails.append( dict(mag_estimate.MagStatsException(agasc_id=agasc_id, msg=msg)) @@ -97,10 +97,10 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres for level in trace: for line in level.splitlines(): logger.debug(line) - logger.debug('') + logger.debug("") bar.close() - logger.debug('-' * 80) + logger.debug("-" * 80) try: agasc_stats = Table(agasc_stats) if agasc_stats else None @@ -112,7 +112,7 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres fails.append( dict( mag_estimate.MagStatsException( - msg=f'Exception at end of get_agasc_id_stats: {str(e)}' + msg=f"Exception at end of get_agasc_id_stats: {str(e)}" ) ) ) @@ -145,7 +145,7 @@ def get_agasc_id_stats_pool( jobs = [] args = [] finished = 0 - logger.info(f'Processing {batch_size} stars per job') + logger.info(f"Processing {batch_size} stars per job") for i in range(0, len(agasc_ids), batch_size): args.append(agasc_ids[i : i + batch_size]) with Pool() as pool: @@ -155,7 +155,7 @@ def get_agasc_id_stats_pool( get_agasc_id_stats, [arg, obs_status_override, tstop, True] ) ) - bar = tqdm(total=len(jobs), desc='progress', disable=no_progress, unit='job') + bar = tqdm(total=len(jobs), desc="progress", disable=no_progress, unit="job") while finished < len(jobs): finished = sum([f.ready() for f in jobs]) if finished - bar.n: @@ -174,7 +174,7 @@ def get_agasc_id_stats_pool( fails.append( dict( mag_estimate.MagStatsException( - agasc_id=agasc_id, msg=f'Failed job: {e}' + agasc_id=agasc_id, msg=f"Failed job: {e}" ) ) ) @@ -194,9 +194,9 @@ def _update_table(table_old, table_new, keys): # checking names, because actual types change upon saving in fits format if set(table_old.colnames) != set(table_new.colnames): raise Exception( - 'Tables have different columns:' - f'\n {table_old.colnames}' - f'\n {table_new.colnames}' + "Tables have different columns:" + f"\n {table_old.colnames}" + f"\n {table_new.colnames}" ) table_old = table_old.copy() new_row = np.ones(len(table_new), dtype=bool) @@ -209,7 +209,7 @@ def _update_table(table_old, table_new, keys): return table.vstack([table_old, table_new[new_row][columns]]) -def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): +def update_mag_stats(obs_stats, agasc_stats, fails, outdir="."): """ Update the mag_stats catalog. @@ -225,11 +225,11 @@ def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): :return: """ if agasc_stats is not None and len(agasc_stats): - filename = outdir / 'mag_stats_agasc.fits' - logger.debug(f'Updating {filename}') + filename = outdir / "mag_stats_agasc.fits" + logger.debug(f"Updating {filename}") if filename.exists(): agasc_stats = _update_table( - table.Table.read(filename), agasc_stats, keys=['agasc_id'] + table.Table.read(filename), agasc_stats, keys=["agasc_id"] ) os.remove(filename) for column in agasc_stats.colnames: @@ -239,11 +239,11 @@ def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): ] agasc_stats.write(filename) if obs_stats is not None and len(obs_stats): - filename = outdir / 'mag_stats_obsid.fits' - logger.debug(f'Updating {filename}') + filename = outdir / "mag_stats_obsid.fits" + logger.debug(f"Updating {filename}") if filename.exists(): obs_stats = _update_table( - table.Table.read(filename), obs_stats, keys=['agasc_id', 'obsid'] + table.Table.read(filename), obs_stats, keys=["agasc_id", "obsid"] ) os.remove(filename) for column in obs_stats.colnames: @@ -251,10 +251,10 @@ def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): obs_stats[column].description = mag_estimate.OBS_STATS_INFO[column] obs_stats.write(filename) if len(fails): - filename = outdir / 'mag_stats_fails.pkl' - logger.info(f'A summary of all failures is saved in {filename}') + filename = outdir / "mag_stats_fails.pkl" + logger.info(f"A summary of all failures is saved in {filename}") # logger.debug(f'Updating {filename}') - with open(filename, 'wb') as out: + with open(filename, "wb") as out: pickle.dump(fails, out) @@ -273,19 +273,19 @@ def update_supplement(agasc_stats, filename, include_all=True): return [], [] if include_all: - outliers_new = agasc_stats[(agasc_stats['n_obsids_ok'] > 0)] + outliers_new = agasc_stats[(agasc_stats["n_obsids_ok"] > 0)] else: outliers_new = agasc_stats[ - (agasc_stats['n_obsids_ok'] > 0) + (agasc_stats["n_obsids_ok"] > 0) & ( - agasc_stats['selected_atol'] - | agasc_stats['selected_rtol'] - | agasc_stats['selected_color'] - | agasc_stats['selected_mag_aca_err'] + agasc_stats["selected_atol"] + | agasc_stats["selected_rtol"] + | agasc_stats["selected_color"] + | agasc_stats["selected_mag_aca_err"] ) ] - outliers_new['mag_aca'] = outliers_new['mag_obs'] - outliers_new['mag_aca_err'] = outliers_new['mag_obs_err'] + outliers_new["mag_aca"] = outliers_new["mag_obs"] + outliers_new["mag_aca_err"] = outliers_new["mag_obs_err"] outliers_new = outliers_new[MAGS_DTYPE.names].as_array() if outliers_new.dtype != MAGS_DTYPE: @@ -296,36 +296,36 @@ def update_supplement(agasc_stats, filename, include_all=True): updated_stars = None if filename.exists(): # I could do what follows directly in place, but the table is not that large. - with tables.File(filename, 'r') as h5: - if 'mags' in h5.root: + with tables.File(filename, "r") as h5: + if "mags" in h5.root: outliers_current = h5.root.mags[:] # find the indices of agasc_ids in both current and new lists _, i_new, i_cur = np.intersect1d( - outliers_new['agasc_id'], - outliers_current['agasc_id'], + outliers_new["agasc_id"], + outliers_current["agasc_id"], return_indices=True, ) current = outliers_current[i_cur] new = outliers_new[i_new] # from those, find the ones which differ in last observation time - i_cur = i_cur[current['last_obs_time'] != new['last_obs_time']] - i_new = i_new[current['last_obs_time'] != new['last_obs_time']] + i_cur = i_cur[current["last_obs_time"] != new["last_obs_time"]] + i_new = i_new[current["last_obs_time"] != new["last_obs_time"]] # overwrite current values with new values (and calculate diff to return) updated_stars = np.zeros(len(outliers_new[i_new]), dtype=MAGS_DTYPE) - updated_stars['mag_aca'] = ( - outliers_new[i_new]['mag_aca'] - outliers_current[i_cur]['mag_aca'] + updated_stars["mag_aca"] = ( + outliers_new[i_new]["mag_aca"] - outliers_current[i_cur]["mag_aca"] ) - updated_stars['mag_aca_err'] = ( - outliers_new[i_new]['mag_aca_err'] - - outliers_current[i_cur]['mag_aca_err'] + updated_stars["mag_aca_err"] = ( + outliers_new[i_new]["mag_aca_err"] + - outliers_current[i_cur]["mag_aca_err"] ) - updated_stars['agasc_id'] = outliers_new[i_new]['agasc_id'] + updated_stars["agasc_id"] = outliers_new[i_new]["agasc_id"] outliers_current[i_cur] = outliers_new[i_new] # find agasc_ids in new list but not in current list new_stars = ~np.in1d( - outliers_new['agasc_id'], outliers_current['agasc_id'] + outliers_new["agasc_id"], outliers_current["agasc_id"] ) # and add them to the current list outliers_current = np.concatenate( @@ -333,20 +333,20 @@ def update_supplement(agasc_stats, filename, include_all=True): ) outliers = np.sort(outliers_current) - new_stars = outliers_new[new_stars]['agasc_id'] + new_stars = outliers_new[new_stars]["agasc_id"] if outliers is None: logger.warning('Creating new "mags" table') outliers = outliers_new - new_stars = outliers_new['agasc_id'] + new_stars = outliers_new["agasc_id"] updated_stars = np.array([], dtype=MAGS_DTYPE) - mode = 'r+' if filename.exists() else 'w' + mode = "r+" if filename.exists() else "w" with tables.File(filename, mode) as h5: - if 'mags' in h5.root: - h5.remove_node('/mags') - h5.create_table('/', 'mags', outliers) - save_version(filename, 'mags') + if "mags" in h5.root: + h5.remove_node("/mags") + h5.create_table("/", "mags", outliers) + save_version(filename, "mags") return new_stars, updated_stars @@ -354,37 +354,37 @@ def update_supplement(agasc_stats, filename, include_all=True): def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): obs = [] if obs_stats and len(obs_stats): - obs_stats = obs_stats[~obs_stats['obs_ok']] - mp_starcat_times = np.unique(obs_stats['mp_starcat_time']) + obs_stats = obs_stats[~obs_stats["obs_ok"]] + mp_starcat_times = np.unique(obs_stats["mp_starcat_time"]) for mp_starcat_time in mp_starcat_times: - rows = obs_stats[obs_stats['mp_starcat_time'] == mp_starcat_time] - rows.sort(keys='agasc_id') + rows = obs_stats[obs_stats["mp_starcat_time"] == mp_starcat_time] + rows.sort(keys="agasc_id") obs.append( { - 'mp_starcat_time': mp_starcat_time, - 'obsid': obs_stats['obsid'], - 'agasc_id': list(rows['agasc_id']), - 'status': 1, - 'comments': obs_stats['comment'], + "mp_starcat_time": mp_starcat_time, + "obsid": obs_stats["obsid"], + "agasc_id": list(rows["agasc_id"]), + "status": 1, + "comments": obs_stats["comment"], } ) for fail in fails: - if fail['agasc_id'] is None or fail['mp_starcat_time'] is None: + if fail["agasc_id"] is None or fail["mp_starcat_time"] is None: continue mp_starcat_times = ( - fail['mp_starcat_time'] - if type(fail['mp_starcat_time']) is list - else [fail['mp_starcat_time']] + fail["mp_starcat_time"] + if type(fail["mp_starcat_time"]) is list + else [fail["mp_starcat_time"]] ) - agasc_id = fail['agasc_id'] + agasc_id = fail["agasc_id"] for mp_starcat_time in mp_starcat_times: obs.append( { - 'mp_starcat_time': mp_starcat_time, - 'obsid': fail['obsid'], - 'agasc_id': [agasc_id], - 'status': 1, - 'comments': fail['msg'], + "mp_starcat_time": mp_starcat_time, + "obsid": fail["obsid"], + "agasc_id": [agasc_id], + "status": 1, + "comments": fail["msg"], } ) if len(obs) == 0: @@ -394,15 +394,15 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): agasc_ids = [] for o in obs: - cat = get_starcheck_catalog(o['obsid']) + cat = get_starcheck_catalog(o["obsid"]) if cat: - cat = cat['cat'] - maxmags = dict(zip(cat['id'], cat['maxmag'])) + cat = cat["cat"] + maxmags = dict(zip(cat["id"], cat["maxmag"])) agasc_ids += [ - (agasc_id, maxmags.get(agasc_id, -1)) for agasc_id in o['agasc_id'] + (agasc_id, maxmags.get(agasc_id, -1)) for agasc_id in o["agasc_id"] ] else: - agasc_ids += [(agasc_id, -1) for agasc_id in obs['agasc_id']] + agasc_ids += [(agasc_id, -1) for agasc_id in obs["agasc_id"]] agasc_ids = dict(sorted(agasc_ids)) @@ -458,7 +458,7 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): tpl = jinja2.Template(yaml_template) result = tpl.render(observations=obs, agasc_ids=agasc_ids) if filename: - with open(filename, 'w') as fh: + with open(filename, "w") as fh: fh.write(result) return result @@ -475,7 +475,7 @@ def do( include_bad=False, dry_run=False, no_progress=None, - email='', + email="", ): """ @@ -509,7 +509,7 @@ def do( # as ascii. It displays a warning which I want to avoid: warnings.filterwarnings("ignore", category=tables.exceptions.FlavorWarning) - filename = output_dir / 'agasc_supplement.h5' + filename = output_dir / "agasc_supplement.h5" if multi_process: get_stats = partial(get_agasc_id_stats_pool, batch_size=10) @@ -518,17 +518,17 @@ def do( skip = True if agasc_ids is None: - obs_in_time = (star_obs_catalogs.STARS_OBS['mp_starcat_time'] >= start) & ( - star_obs_catalogs.STARS_OBS['mp_starcat_time'] <= stop + obs_in_time = (star_obs_catalogs.STARS_OBS["mp_starcat_time"] >= start) & ( + star_obs_catalogs.STARS_OBS["mp_starcat_time"] <= stop ) - agasc_ids = sorted(star_obs_catalogs.STARS_OBS[obs_in_time]['agasc_id']) + agasc_ids = sorted(star_obs_catalogs.STARS_OBS[obs_in_time]["agasc_id"]) else: - agasc_ids = np.intersect1d(agasc_ids, star_obs_catalogs.STARS_OBS['agasc_id']) + agasc_ids = np.intersect1d(agasc_ids, star_obs_catalogs.STARS_OBS["agasc_id"]) skip = False agasc_ids = np.unique(agasc_ids) stars_obs = star_obs_catalogs.STARS_OBS[ - np.in1d(star_obs_catalogs.STARS_OBS['agasc_id'], agasc_ids) + np.in1d(star_obs_catalogs.STARS_OBS["agasc_id"], agasc_ids) ] # if supplement exists: @@ -539,75 +539,75 @@ def do( # - include only the ones with supplement.last_obs_time < than stars_obs.mp_starcat_time obs_status_override = {} if filename.exists(): - with tables.File(filename, 'r') as h5: - if not include_bad and 'bad' in h5.root: - logger.info('Excluding bad stars') + with tables.File(filename, "r") as h5: + if not include_bad and "bad" in h5.root: + logger.info("Excluding bad stars") stars_obs = stars_obs[ - ~np.in1d(stars_obs['agasc_id'], h5.root.bad[:]['agasc_id']) + ~np.in1d(stars_obs["agasc_id"], h5.root.bad[:]["agasc_id"]) ] - if 'obs' in h5.root: + if "obs" in h5.root: obs_status_override = table.Table(h5.root.obs[:]) obs_status_override.convert_bytestring_to_unicode() obs_status_override = { - (r['mp_starcat_time'], r['agasc_id']): { - 'status': r['status'], - 'comments': r['comments'], + (r["mp_starcat_time"], r["agasc_id"]): { + "status": r["status"], + "comments": r["comments"], } for r in obs_status_override } - if 'mags' in h5.root and len(stars_obs): + if "mags" in h5.root and len(stars_obs): outliers_current = h5.root.mags[:] times = ( - stars_obs[['agasc_id', 'mp_starcat_time']] - .group_by('agasc_id') + stars_obs[["agasc_id", "mp_starcat_time"]] + .group_by("agasc_id") .groups.aggregate(lambda d: np.max(CxoTime(d)).date) ) if len(outliers_current): times = table.join( times, - table.Table(outliers_current[['agasc_id', 'last_obs_time']]), - join_type='left', + table.Table(outliers_current[["agasc_id", "last_obs_time"]]), + join_type="left", ) else: - times['last_obs_time'] = table.MaskedColumn( - np.zeros(len(times), dtype=h5.root.mags.dtype['last_obs_time']), + times["last_obs_time"] = table.MaskedColumn( + np.zeros(len(times), dtype=h5.root.mags.dtype["last_obs_time"]), mask=np.ones(len(times), dtype=bool), ) if skip: - if hasattr(times['last_obs_time'], 'mask'): + if hasattr(times["last_obs_time"], "mask"): # the mask exists if there are stars in stars_obs # that are not in outliers_current - update = times['last_obs_time'].mask | ( - (~times['last_obs_time'].mask) + update = times["last_obs_time"].mask | ( + (~times["last_obs_time"].mask) & ( - CxoTime(times['mp_starcat_time']).cxcsec - > times['last_obs_time'] + CxoTime(times["mp_starcat_time"]).cxcsec + > times["last_obs_time"] ).data ) else: update = ( - CxoTime(times['mp_starcat_time']).cxcsec - > times['last_obs_time'] + CxoTime(times["mp_starcat_time"]).cxcsec + > times["last_obs_time"] ) stars_obs = stars_obs[ - np.in1d(stars_obs['agasc_id'], times[update]['agasc_id']) + np.in1d(stars_obs["agasc_id"], times[update]["agasc_id"]) ] - agasc_ids = np.sort(np.unique(stars_obs['agasc_id'])) + agasc_ids = np.sort(np.unique(stars_obs["agasc_id"])) if len(update) - np.sum(update): logger.info( - f'Skipping {len(update) - np.sum(update)} ' - 'stars already in the supplement' + f"Skipping {len(update) - np.sum(update)} " + "stars already in the supplement" ) if len(stars_obs) == 0: - logger.info(f'There are no new observations to process') + logger.info(f"There are no new observations to process") return # do the processing - logger.info(f'Will process {len(agasc_ids)} stars on {len(stars_obs)} observations') - logger.info(f'from {start} to {stop}') + logger.info(f"Will process {len(agasc_ids)} stars on {len(stars_obs)} observations") + logger.info(f"from {start} to {stop}") if dry_run: return @@ -618,20 +618,20 @@ def do( no_progress=no_progress, ) - failed_global = [f for f in fails if not f['agasc_id'] and not f['obsid']] - failed_stars = [f for f in fails if f['agasc_id'] and not f['obsid']] - failed_obs = [f for f in fails if f['obsid']] + failed_global = [f for f in fails if not f["agasc_id"] and not f["obsid"]] + failed_stars = [f for f in fails if f["agasc_id"] and not f["obsid"]] + failed_obs = [f for f in fails if f["obsid"]] msg = ( - 'Got:\n' - f' {0 if obs_stats is None else len(obs_stats)} OBSIDs,' - f' {0 if agasc_stats is None else len(agasc_stats)} stars,' + "Got:\n" + f" {0 if obs_stats is None else len(obs_stats)} OBSIDs," + f" {0 if agasc_stats is None else len(agasc_stats)} stars," ) if failed_obs: - msg += f' {len(failed_obs)} failed observations,' + msg += f" {len(failed_obs)} failed observations," if failed_stars: - msg += f' {len(failed_stars)} failed stars,' + msg += f" {len(failed_stars)} failed stars," if failed_global: - msg += f' {len(failed_global)} global errors' + msg += f" {len(failed_global)} global errors" logger.info(msg) if not output_dir.exists(): @@ -639,78 +639,78 @@ def do( update_mag_stats(obs_stats, agasc_stats, fails, output_dir) - obs_status_file = output_dir / 'obs_status.yml' + obs_status_file = output_dir / "obs_status.yml" try: write_obs_status_yaml( [], fails=failed_obs + failed_stars, filename=obs_status_file ) except Exception as e: - logger.warning(f'Failed to write {obs_status_file}: {e}') + logger.warning(f"Failed to write {obs_status_file}: {e}") new_stars, updated_stars = update_supplement(agasc_stats, filename=filename) - logger.info(f' {len(new_stars)} new stars, {len(updated_stars)} updated stars') + logger.info(f" {len(new_stars)} new stars, {len(updated_stars)} updated stars") if agasc_stats is not None and len(agasc_stats): if email: try: bad_obs = ( - (obs_stats['mp_starcat_time'] >= start) - & (obs_stats['mp_starcat_time'] < stop) - & ~obs_stats['obs_ok'] + (obs_stats["mp_starcat_time"] >= start) + & (obs_stats["mp_starcat_time"] < stop) + & ~obs_stats["obs_ok"] ) if np.any(bad_obs): msr.email_bad_obs_report(obs_stats[bad_obs], to=email) except Exception as e: - logger.error(f'Error sending email to {email}: {e}') + logger.error(f"Error sending email to {email}: {e}") if report and len(agasc_stats): if report_date is None: report_dir = reports_dir - report_data_file = report_dir / f'report_data.pkl' + report_data_file = report_dir / f"report_data.pkl" nav_links = None report_date = CxoTime.now() else: - report_dir = reports_dir / f'{report_date.date[:8]}' - report_data_file = report_dir / f'report_data_{report_date.date[:8]}.pkl' + report_dir = reports_dir / f"{report_date.date[:8]}" + report_data_file = report_dir / f"report_data_{report_date.date[:8]}.pkl" week = time.TimeDelta(7 * u.day) nav_links = { - 'previous': f'../{(report_date - week).date[:8]}/index.html', - 'up': '..', - 'next': f'../{(report_date + week).date[:8]}/index.html', + "previous": f"../{(report_date - week).date[:8]}/index.html", + "up": "..", + "next": f"../{(report_date + week).date[:8]}/index.html", } # If the report data file exists, the arguments for the report from the file are # modified according to the current run. Otherwise, they are created from scratch. if report_data_file.exists(): - with open(report_data_file, 'rb') as fh: + with open(report_data_file, "rb") as fh: report_data = pickle.load(fh) - logger.info(f'Loading existing report data from {report_data_file}') - multi_star_html_args = report_data['args'] + logger.info(f"Loading existing report data from {report_data_file}") + multi_star_html_args = report_data["args"] # arguments for the report are modified here # merge fails: # - from previous run, take fails that were not run just now # - add current fails - multi_star_html_args['fails'] = fails - multi_star_html_args['no_progress'] = no_progress + multi_star_html_args["fails"] = fails + multi_star_html_args["no_progress"] = no_progress else: sections = [ - {'id': 'new_stars', 'title': 'New Stars', 'stars': new_stars}, + {"id": "new_stars", "title": "New Stars", "stars": new_stars}, { - 'id': 'updated_stars', - 'title': 'Updated Stars', - 'stars': updated_stars['agasc_id'] if len(updated_stars) else [], + "id": "updated_stars", + "title": "Updated Stars", + "stars": updated_stars["agasc_id"] if len(updated_stars) else [], }, { - 'id': 'other_stars', - 'title': 'Other Stars', - 'stars': list( - agasc_stats['agasc_id'][ - ~np.in1d(agasc_stats['agasc_id'], new_stars) + "id": "other_stars", + "title": "Other Stars", + "stars": list( + agasc_stats["agasc_id"][ + ~np.in1d(agasc_stats["agasc_id"], new_stars) & ~np.in1d( - agasc_stats['agasc_id'], updated_stars['agasc_id'] + agasc_stats["agasc_id"], updated_stars["agasc_id"] ) ] ), @@ -718,7 +718,7 @@ def do( ] multi_star_html_args = dict( - filename='index.html', + filename="index.html", sections=sections, updated_stars=updated_stars, fails=fails, @@ -732,12 +732,12 @@ def do( try: report = msr.MagEstimateReport( - agasc_stats=output_dir / 'mag_stats_agasc.fits', - obs_stats=output_dir / 'mag_stats_obsid.fits', + agasc_stats=output_dir / "mag_stats_agasc.fits", + obs_stats=output_dir / "mag_stats_obsid.fits", directory=report_dir, ) report.multi_star_html(**multi_star_html_args) - latest = reports_dir / 'latest' + latest = reports_dir / "latest" if os.path.lexists(latest): logger.debug('Removing existing "latest" symlink') latest.unlink() @@ -745,24 +745,24 @@ def do( latest.symlink_to(report_dir.absolute()) except Exception as e: report_dir = output_dir - logger.error(f'Error when creating report: {e}') + logger.error(f"Error when creating report: {e}") exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is not None: trace = traceback.format_exception(exc_type, exc_value, exc_traceback) for level in trace: for line in level.splitlines(): logger.debug(line) - logger.debug('') + logger.debug("") finally: report_data_file = report_dir / report_data_file.name if not report_dir.exists(): report_dir.mkdir(parents=True) - report_data = {'args': multi_star_html_args, 'directory': report_dir} - with open(report_data_file, 'wb') as fh: + report_data = {"args": multi_star_html_args, "directory": report_dir} + with open(report_data_file, "wb") as fh: pickle.dump(report_data, fh) - logger.info(f'Report data saved in {report_data_file}') + logger.info(f"Report data saved in {report_data_file}") elif len(agasc_stats) == 0: - logger.info('Nothing to report (no stars)') + logger.info("Nothing to report (no stars)") now = datetime.datetime.now() logger.info(f"done at {now}") diff --git a/agasc/supplement/utils.py b/agasc/supplement/utils.py index 7c1a0ec9..f5214d0e 100644 --- a/agasc/supplement/utils.py +++ b/agasc/supplement/utils.py @@ -13,59 +13,59 @@ __all__ = [ - 'get_supplement_table', - 'save_version', - 'update_mags_table', - 'update_obs_table', - 'add_bad_star', + "get_supplement_table", + "save_version", + "update_mags_table", + "update_obs_table", + "add_bad_star", ] -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") -AGASC_SUPPLEMENT_TABLES = ('mags', 'bad', 'obs', 'last_updated', 'agasc_versions') +AGASC_SUPPLEMENT_TABLES = ("mags", "bad", "obs", "last_updated", "agasc_versions") -BAD_DTYPE = np.dtype([('agasc_id', np.int32), ('source', np.int16)]) +BAD_DTYPE = np.dtype([("agasc_id", np.int32), ("source", np.int16)]) MAGS_DTYPE = np.dtype( [ - ('agasc_id', np.int32), - ('mag_aca', np.float32), - ('mag_aca_err', np.float32), - ('last_obs_time', np.float64), + ("agasc_id", np.int32), + ("mag_aca", np.float32), + ("mag_aca_err", np.float32), + ("last_obs_time", np.float64), ] ) OBS_DTYPE = np.dtype( [ - ('mp_starcat_time', ' 0 else 1 + dat = _get_table(filename, "bad", BAD_DTYPE, create=create) + default = dat["source"].max() + 1 if len(dat) > 0 else 1 bad = [ [agasc_id, default if source is None else source] for agasc_id, source in bad @@ -332,25 +332,25 @@ def add_bad_star(filename, bad, dry_run=False, create=False): update = False for agasc_id, source in zip(bad_star_ids, bad_star_source): - if agasc_id not in dat['agasc_id']: + if agasc_id not in dat["agasc_id"]: dat.add_row((agasc_id, source)) - logger.info(f'Appending {agasc_id=} with {source=}') + logger.info(f"Appending {agasc_id=} with {source=}") update = True if not update: return - logger.info('') - logger.info('IMPORTANT:') - logger.info('Edit following if source ID is new:') + logger.info("") + logger.info("IMPORTANT:") + logger.info("Edit following if source ID is new:") logger.info( - ' https://github.com/sot/agasc/wiki/Add-bad-star-to-AGASC-supplement-manually' + " https://github.com/sot/agasc/wiki/Add-bad-star-to-AGASC-supplement-manually" ) - logger.info('') - logger.info('The wiki page also includes instructions for test, review, approval') - logger.info('and installation.') + logger.info("") + logger.info("The wiki page also includes instructions for test, review, approval") + logger.info("and installation.") if not dry_run: - dat.write(str(filename), format='hdf5', path='bad', append=True, overwrite=True) - save_version(filename, 'bad') + dat.write(str(filename), format="hdf5", path="bad", append=True, overwrite=True) + save_version(filename, "bad") def update_obs_table(filename, obs, dry_run=False, create=False): @@ -380,9 +380,9 @@ def update_obs_table(filename, obs, dry_run=False, create=False): update_table( filename, obs, - 'obs', + "obs", OBS_DTYPE, - keys=['agasc_id', 'mp_starcat_time'], + keys=["agasc_id", "mp_starcat_time"], dry_run=dry_run, create=create, ) @@ -408,9 +408,9 @@ def update_mags_table(filename, mags, dry_run=False, create=False): update_table( filename, mags, - 'mags', + "mags", MAGS_DTYPE, - keys=['agasc_id'], + keys=["agasc_id"], dry_run=dry_run, create=create, ) diff --git a/agasc/tests/test_agasc_1.py b/agasc/tests/test_agasc_1.py index e630ca34..cbfb8f4c 100644 --- a/agasc/tests/test_agasc_1.py +++ b/agasc/tests/test_agasc_1.py @@ -20,20 +20,20 @@ def test_multi_agasc(): with tables.open_file(agasc.default_agasc_file()) as h5: middle = int(len(h5.root.data) // 2) stars1 = Table(h5.root.data[middle : middle + 20]) - stars1.write(os.path.join(tempdir, 'stars1.h5'), path='data') + stars1.write(os.path.join(tempdir, "stars1.h5"), path="data") stars2 = Table(h5.root.data[middle + 20 : middle + 60]) - stars2.write(os.path.join(tempdir, 'stars2.h5'), path='data') + stars2.write(os.path.join(tempdir, "stars2.h5"), path="data") # Fetch all the stars from a custom agasc and make sure we have the right number of stars # with no errors all_stars2 = agasc.get_agasc_cone( - 0, 90, radius=180, agasc_file=os.path.join(tempdir, 'stars2.h5') + 0, 90, radius=180, agasc_file=os.path.join(tempdir, "stars2.h5") ) assert len(all_stars2) == len(stars2) # Fetch all the stars from the other custom agasc and do the same. The point of the two files # is to confirm that the caching behavior in agasc doesn't cause problems with fetches all_stars1 = agasc.get_agasc_cone( - 0, 90, radius=180, agasc_file=os.path.join(tempdir, 'stars1.h5') + 0, 90, radius=180, agasc_file=os.path.join(tempdir, "stars1.h5") ) assert len(all_stars1) == len(stars1) @@ -41,17 +41,17 @@ def test_multi_agasc(): # we get the same star from the reference agasc. Do this with the stars2 file as this confirms # that we can switch back and forth between files and get the correct content. cone2 = agasc.get_agasc_cone( - all_stars2['RA'][0], - all_stars2['DEC'][0], + all_stars2["RA"][0], + all_stars2["DEC"][0], radius=0.000001, - agasc_file=os.path.join(tempdir, 'stars2.h5'), + agasc_file=os.path.join(tempdir, "stars2.h5"), ) # And this is a read of the default agasc file after the custom ones so should confirm that # the custom files didn't break that access. cone2_full = agasc.get_agasc_cone( - all_stars2['RA'][0], all_stars2['DEC'][0], radius=0.000001 + all_stars2["RA"][0], all_stars2["DEC"][0], radius=0.000001 ) - assert cone2[0]['AGASC_ID'] == cone2_full[0]['AGASC_ID'] + assert cone2[0]["AGASC_ID"] == cone2_full[0]["AGASC_ID"] # Confirm that there is just one star in this test setup (not a module test, but confirms test # setup is as intended). assert len(cone2_full) == 1 @@ -70,11 +70,11 @@ def test_update_color1_func(): # Fourth now gets COLOR1 = 1.499 because RSV3=1 => good mag # Fifth is still 1.5 because RSV3=0 (no good mag available so still "bad mag") # Sixth now gets COLOR1 = COLOR2 * 0.850 = 2.0 - stars = Table([color1, color2, rsv3], names=['COLOR1', 'COLOR2', 'RSV3']) + stars = Table([color1, color2, rsv3], names=["COLOR1", "COLOR2", "RSV3"]) update_color1_column(stars) - assert np.allclose(stars['COLOR1'], [1.0, 1.0, 1.5, 1.499, 1.5, 2.0]) - assert np.allclose(stars['COLOR2'], color2) + assert np.allclose(stars["COLOR1"], [1.0, 1.0, 1.5, 1.499, 1.5, 2.0]) + assert np.allclose(stars["COLOR2"], color2) def test_update_color1_get_star(): @@ -89,16 +89,16 @@ def test_update_color1_get_star(): """ star = agasc.get_star(981997696) - assert np.isclose(star['COLOR1'], 1.5) + assert np.isclose(star["COLOR1"], 1.5) star = agasc.get_star(759439648) - assert np.isclose(star['COLOR1'], 1.499) + assert np.isclose(star["COLOR1"], 1.499) star = agasc.get_star(981997696, fix_color1=False) - assert np.isclose(star['COLOR1'], 1.5) + assert np.isclose(star["COLOR1"], 1.5) star = agasc.get_star(759439648, fix_color1=False) - assert np.isclose(star['COLOR1'], 1.5) + assert np.isclose(star["COLOR1"], 1.5) def test_update_color1_get_agasc_cone(): @@ -107,14 +107,14 @@ def test_update_color1_get_agasc_cone(): """ ra, dec = 323.22831196, -13.11621348 stars = agasc.get_agasc_cone(ra, dec, 0.2) - stars.add_index('AGASC_ID') - assert np.isclose(stars.loc[759960152]['COLOR1'], 1.60055, rtol=0, atol=0.0005) - assert np.isclose(stars.loc[759439648]['COLOR1'], 1.499, rtol=0, atol=0.0005) + stars.add_index("AGASC_ID") + assert np.isclose(stars.loc[759960152]["COLOR1"], 1.60055, rtol=0, atol=0.0005) + assert np.isclose(stars.loc[759439648]["COLOR1"], 1.499, rtol=0, atol=0.0005) stars = agasc.get_agasc_cone(ra, dec, 0.2, fix_color1=False) - stars.add_index('AGASC_ID') - assert np.isclose(stars.loc[759960152]['COLOR1'], 1.5, rtol=0, atol=0.0005) - assert np.isclose(stars.loc[759439648]['COLOR1'], 1.5, rtol=0, atol=0.0005) + stars.add_index("AGASC_ID") + assert np.isclose(stars.loc[759960152]["COLOR1"], 1.5, rtol=0, atol=0.0005) + assert np.isclose(stars.loc[759439648]["COLOR1"], 1.5, rtol=0, atol=0.0005) def test_get_agasc_filename(tmp_path, monkeypatch): diff --git a/agasc/tests/test_agasc_2.py b/agasc/tests/test_agasc_2.py index b7cfe58e..a399741d 100644 --- a/agasc/tests/test_agasc_2.py +++ b/agasc/tests/test_agasc_2.py @@ -32,30 +32,30 @@ import agasc -os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = 'False' +os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = "False" # See if we can get to ASCDS environment and mp_get_agasc try: - ascrc_file = '{}/.ascrc'.format(os.environ['HOME']) + ascrc_file = "{}/.ascrc".format(os.environ["HOME"]) assert os.path.exists(ascrc_file) ascds_env = Ska.Shell.getenv( - 'source {} -r release'.format(ascrc_file), shell='tcsh' + "source {} -r release".format(ascrc_file), shell="tcsh" ) - assert 'ASCDS_BIN' in ascds_env - cmd = 'mp_get_agasc -r 10 -d 20 -w 0.01' + assert "ASCDS_BIN" in ascds_env + cmd = "mp_get_agasc -r 10 -d 20 -w 0.01" # Run the command to check for bad status (which will throw exception) - Ska.Shell.run_shell(cmd, shell='bash', env=ascds_env) - match = re.search(r'agasc([p0-9]+)', ascds_env['ASCDS_AGASC']) + Ska.Shell.run_shell(cmd, shell="bash", env=ascds_env) + match = re.search(r"agasc([p0-9]+)", ascds_env["ASCDS_AGASC"]) DS_AGASC_VERSION = match.group(1) except Exception: ascds_env = None DS_AGASC_VERSION = None -NO_MAGS_IN_SUPPLEMENT = not any(agasc.get_supplement_table('mags', as_dict=True)) -NO_OBS_IN_SUPPLEMENT = not any(agasc.get_supplement_table('obs', as_dict=True)) +NO_MAGS_IN_SUPPLEMENT = not any(agasc.get_supplement_table("mags", as_dict=True)) +NO_OBS_IN_SUPPLEMENT = not any(agasc.get_supplement_table("obs", as_dict=True)) -HAS_KSH = os.path.exists('/bin/ksh') # dependency of mp_get_agascid +HAS_KSH = os.path.exists("/bin/ksh") # dependency of mp_get_agascid AGASC_COL_DESCR = """ AGASC_ID - a unique long integer used for identification. @@ -118,39 +118,39 @@ TEST_ASCDS = False else: try: - agasc.get_agasc_filename('miniagasc_*', version=DS_AGASC_VERSION) + agasc.get_agasc_filename("miniagasc_*", version=DS_AGASC_VERSION) except FileNotFoundError: TEST_ASCDS = False else: TEST_ASCDS = True # Latest full release of miniagasc -MINIAGASC = agasc.get_agasc_filename('miniagasc_*') +MINIAGASC = agasc.get_agasc_filename("miniagasc_*") def get_ds_agasc_cone(ra, dec): - cmd = 'mp_get_agasc -r {!r} -d {!r} -w {!r}'.format(ra, dec, TEST_RADIUS) + cmd = "mp_get_agasc -r {!r} -d {!r} -w {!r}".format(ra, dec, TEST_RADIUS) lines = Ska.Shell.tcsh(cmd, env=ascds_env) dat = ascii.read(lines, Reader=ascii.NoHeader, names=AGASC_COLNAMES) - ok1 = agasc.sphere_dist(ra, dec, dat['RA'], dat['DEC']) <= TEST_RADIUS - ok2 = dat['MAG_ACA'] - 3.0 * dat['MAG_ACA_ERR'] / 100.0 < 11.5 + ok1 = agasc.sphere_dist(ra, dec, dat["RA"], dat["DEC"]) <= TEST_RADIUS + ok2 = dat["MAG_ACA"] - 3.0 * dat["MAG_ACA_ERR"] / 100.0 < 11.5 dat = dat[ok1 & ok2] - if os.environ.get('WRITE_AGASC_TEST_FILES'): + if os.environ.get("WRITE_AGASC_TEST_FILES"): version = DS_AGASC_VERSION test_file = get_test_file(ra, dec, version) - print(f'\nWriting {test_file} based on mp_get_agasc\n') - dat.write(test_file, format='fits') + print(f"\nWriting {test_file} based on mp_get_agasc\n") + dat.write(test_file, format="fits") return dat def get_test_file(ra, dec, version): - return TEST_DIR / 'data' / f'ref_ra_{ra}_dec_{dec}_{version}.fits.gz' + return TEST_DIR / "data" / f"ref_ra_{ra}_dec_{dec}_{version}.fits.gz" -def get_reference_agasc_values(ra, dec, version='1p7'): +def get_reference_agasc_values(ra, dec, version="1p7"): dat = Table.read(get_test_file(ra, dec, version)) return dat @@ -161,7 +161,7 @@ def get_reference_agasc_values(ra, dec, version='1p7'): # mp_get_agasc not accounting for proper motion. -@pytest.mark.parametrize("version", ['1p6', '1p7']) +@pytest.mark.parametrize("version", ["1p6", "1p7"]) @pytest.mark.parametrize("ra,dec", list(zip(RAS, DECS))) def test_agasc_conesearch(ra, dec, version): """ @@ -171,24 +171,24 @@ def test_agasc_conesearch(ra, dec, version): try: ref_stars = get_reference_agasc_values(ra, dec, version=version) except FileNotFoundError: - if os.environ.get('WRITE_AGASC_TEST_FILES'): + if os.environ.get("WRITE_AGASC_TEST_FILES"): ref_stars = agasc.get_agasc_cone( ra, dec, radius=TEST_RADIUS, - agasc_file=agasc.get_agasc_filename('miniagasc_*', version=version), - date='2000:001', + agasc_file=agasc.get_agasc_filename("miniagasc_*", version=version), + date="2000:001", fix_color1=False, ) test_file = get_test_file(ra, dec, version) - print(f'\nWriting {test_file} based on miniagasc\n') - ref_stars.write(test_file, format='fits') - pytest.skip('Reference data unavailable') + print(f"\nWriting {test_file} based on miniagasc\n") + ref_stars.write(test_file, format="fits") + pytest.skip("Reference data unavailable") else: _test_agasc(ra, dec, ref_stars, version) -@pytest.mark.skipif('not TEST_ASCDS') +@pytest.mark.skipif("not TEST_ASCDS") @pytest.mark.parametrize("ra,dec", list(zip(RAS, DECS))) def test_against_ds_agasc(ra, dec): """ @@ -199,28 +199,28 @@ def test_against_ds_agasc(ra, dec): _test_agasc(ra, dec, ref_stars, version=DS_AGASC_VERSION) -def _test_agasc(ra, dec, ref_stars, version='1p7'): - agasc_file = agasc.get_agasc_filename('miniagasc_*', version=version) +def _test_agasc(ra, dec, ref_stars, version="1p7"): + agasc_file = agasc.get_agasc_filename("miniagasc_*", version=version) stars1 = agasc.get_agasc_cone( ra, dec, radius=TEST_RADIUS, agasc_file=agasc_file, - date='2000:001', + date="2000:001", fix_color1=False, ) - stars1.sort('AGASC_ID') + stars1.sort("AGASC_ID") stars2 = ref_stars.copy() - stars2.sort('AGASC_ID') + stars2.sort("AGASC_ID") # First make sure that the common stars are identical - agasc_ids = set(stars1['AGASC_ID']).intersection(set(stars2['AGASC_ID'])) + agasc_ids = set(stars1["AGASC_ID"]).intersection(set(stars2["AGASC_ID"])) for agasc_id in agasc_ids: - star1 = stars1[np.searchsorted(stars1['AGASC_ID'], agasc_id)] - star2 = stars2[np.searchsorted(stars2['AGASC_ID'], agasc_id)] + star1 = stars1[np.searchsorted(stars1["AGASC_ID"], agasc_id)] + star2 = stars2[np.searchsorted(stars2["AGASC_ID"], agasc_id)] for colname in AGASC_COLNAMES: - if star1[colname].dtype.kind == 'f': + if star1[colname].dtype.kind == "f": assert np.all(np.abs(star1[colname] - star2[colname]) < 1e-4) else: assert star1[colname] == star2[colname] @@ -228,36 +228,36 @@ def _test_agasc(ra, dec, ref_stars, version='1p7'): # Second make sure that the non-common stars are all just at the edge # of the faint mag limit, due to precision loss in mp_get_agasc for s1, s2 in ((stars1, stars2), (stars2, stars1)): - mm1 = set(s1['AGASC_ID']) - set(s2['AGASC_ID']) + mm1 = set(s1["AGASC_ID"]) - set(s2["AGASC_ID"]) for agasc_id in mm1: - idx = np.flatnonzero(s1['AGASC_ID'] == agasc_id)[0] + idx = np.flatnonzero(s1["AGASC_ID"] == agasc_id)[0] star = s1[idx] bad_is_star1 = s1 is stars1 - rad = agasc.sphere_dist(ra, dec, star['RA'], star['DEC']) - adj_mag = star['MAG_ACA'] - 3.0 * star['MAG_ACA_ERR'] / 100.0 + rad = agasc.sphere_dist(ra, dec, star["RA"], star["DEC"]) + adj_mag = star["MAG_ACA"] - 3.0 * star["MAG_ACA_ERR"] / 100.0 if adj_mag < 11.5 * 0.99: # Allow for loss of precision in output of mp_get_agasc - print('Bad star', agasc_id, rad, adj_mag, bad_is_star1) + print("Bad star", agasc_id, rad, adj_mag, bad_is_star1) assert False def test_basic(): star = agasc.get_star(1180612288) # High-PM star - assert np.isclose(star['RA'], 219.9053773) - assert np.isclose(star['DEC'], -60.8371572) + assert np.isclose(star["RA"], 219.9053773) + assert np.isclose(star["DEC"], -60.8371572) - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], 0.5) - stars.sort('MAG_ACA') + stars = agasc.get_agasc_cone(star["RA"], star["DEC"], 0.5) + stars.sort("MAG_ACA") agasc_ids = [1180612176, 1180612296, 1180612184, 1180612288, 1180612192] mags = [-0.663, -0.576, -0.373, 0.53, 0.667] - assert np.allclose(stars['AGASC_ID'][:5], agasc_ids) - assert np.allclose(stars['MAG_ACA'][:5], mags) + assert np.allclose(stars["AGASC_ID"][:5], agasc_ids) + assert np.allclose(stars["MAG_ACA"][:5], mags) def test_get_stars1(): # First check that get_stars() gives the same as get_star for the scalar case - star1 = agasc.get_star(1180612288, date='2019:001') - star2 = agasc.get_stars(1180612288, dates='2019:001') + star1 = agasc.get_star(1180612288, date="2019:001") + star2 = agasc.get_stars(1180612288, dates="2019:001") assert isinstance(star2, Row) for name in star1.colnames: assert star1[name] == star2[name] @@ -265,9 +265,9 @@ def test_get_stars1(): def test_get_stars2(): """get_stars() broadcasts ids""" - star0 = agasc.get_star(1180612288, date='2010:001') - star1 = agasc.get_star(1180612288, date='2019:001') - star2 = agasc.get_stars(1180612288, dates=['2010:001', '2019:001']) + star0 = agasc.get_star(1180612288, date="2010:001") + star1 = agasc.get_star(1180612288, date="2019:001") + star2 = agasc.get_stars(1180612288, dates=["2010:001", "2019:001"]) for name in star1.colnames: assert star0[name] == star2[0][name] @@ -278,7 +278,7 @@ def test_get_stars3(): agasc_ids = [1180612176, 1180612296, 1180612184, 1180612288, 1180612192] mags = [-0.663, -0.576, -0.373, 0.53, 0.667] stars = agasc.get_stars(agasc_ids) - assert np.allclose(stars['MAG_ACA'], mags) + assert np.allclose(stars["MAG_ACA"], mags) def test_get_stars_many(): @@ -286,14 +286,14 @@ def test_get_stars_many(): from .. import agasc stars = agasc.get_agasc_cone(0, 0, radius=0.5) - agasc_ids = stars['AGASC_ID'] - stars1 = agasc.get_stars(agasc_ids, dates='2020:001') # read_where method + agasc_ids = stars["AGASC_ID"] + stars1 = agasc.get_stars(agasc_ids, dates="2020:001") # read_where method stars2 = agasc.get_stars( - agasc_ids, dates='2020:001', method_threshold=1 + agasc_ids, dates="2020:001", method_threshold=1 ) # read entire AGASC - assert stars1.get_stars_method == 'tables_read_where' - assert stars2.get_stars_method == 'read_entire_agasc' + assert stars1.get_stars_method == "tables_read_where" + assert stars2.get_stars_method == "read_entire_agasc" assert stars1.colnames == stars2.colnames for name in stars1.colnames: @@ -302,7 +302,7 @@ def test_get_stars_many(): def test_float16(): stars = agasc.get_agasc_cone(np.float16(219.90279), np.float16(-60.83358), 0.015) - assert stars['AGASC_ID'][0] == 1180612176 + assert stars["AGASC_ID"][0] == 1180612176 def test_proper_motion(): @@ -310,27 +310,27 @@ def test_proper_motion(): Test that the filtering in get_agasc_cone correctly expands the initial search radius and then does final filtering using PM-corrected positions. """ - star = agasc.get_star(1180612288, date='2017:001') # High-PM star + star = agasc.get_star(1180612288, date="2017:001") # High-PM star radius = 2.0 / 3600 # 5 arcsec - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2000:001') + stars = agasc.get_agasc_cone(star["RA"], star["DEC"], radius, date="2000:001") assert len(stars) == 1 - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2017:001') + stars = agasc.get_agasc_cone(star["RA"], star["DEC"], radius, date="2017:001") assert len(stars) == 0 stars = agasc.get_agasc_cone( - star['RA'], star['DEC'], radius, date='2017:001', pm_filter=False + star["RA"], star["DEC"], radius, date="2017:001", pm_filter=False ) assert len(stars) == 1 stars = agasc.get_agasc_cone( - star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001' + star["RA_PMCORR"], star["DEC_PMCORR"], radius, date="2017:001" ) assert len(stars) == 1 stars = agasc.get_agasc_cone( - star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001', pm_filter=False + star["RA_PMCORR"], star["DEC_PMCORR"], radius, date="2017:001", pm_filter=False ) assert len(stars) == 0 @@ -340,13 +340,13 @@ def test_proper_motion(): [ ( 1180612288, - '2020:001', + "2020:001", 219.864331, -60.831868, "high proper motion, epoch 2000", ), - (198451217, '2020:001', 247.892206, 19.276605, "epoch 1982 star"), - (501219465, '2020:001', 166.998976, 52.822080, "epoch 1984 star"), + (198451217, "2020:001", 247.892206, 19.276605, "epoch 1982 star"), + (501219465, "2020:001", 166.998976, 52.822080, "epoch 1984 star"), ], ) def test_add_pmcorr_is_consistent(agasc_id, date, ra_pmcorr, dec_pmcorr, label): @@ -354,37 +354,37 @@ def test_add_pmcorr_is_consistent(agasc_id, date, ra_pmcorr, dec_pmcorr, label): Check that the proper-motion corrected position is consistent reference/regress values. """ star = agasc.get_star(agasc_id, date=date) - assert np.isclose(star['RA_PMCORR'], ra_pmcorr, rtol=0, atol=1e-5) - assert np.isclose(star['DEC_PMCORR'], dec_pmcorr, rtol=0, atol=1e-5) + assert np.isclose(star["RA_PMCORR"], ra_pmcorr, rtol=0, atol=1e-5) + assert np.isclose(star["DEC_PMCORR"], dec_pmcorr, rtol=0, atol=1e-5) def mp_get_agascid(agasc_id): - cmd = 'mp_get_agascid {!r}'.format(agasc_id) + cmd = "mp_get_agascid {!r}".format(agasc_id) lines = Ska.Shell.tcsh(cmd, env=ascds_env) - lines = [line for line in lines if re.match(r'^\s*\d', line)] + lines = [line for line in lines if re.match(r"^\s*\d", line)] dat = ascii.read(lines, Reader=ascii.NoHeader, names=AGASC_COLNAMES) return dat -@pytest.mark.skipif('not HAS_KSH') -@pytest.mark.skipif('not TEST_ASCDS') +@pytest.mark.skipif("not HAS_KSH") +@pytest.mark.skipif("not TEST_ASCDS") @pytest.mark.parametrize("ra,dec", list(zip(RAS[:2], DECS[:2]))) def test_agasc_id(ra, dec, radius=0.2, nstar_limit=5): agasc_file = agasc.get_agasc_filename("miniagasc_*", version=DS_AGASC_VERSION) - print('ra, dec =', ra, dec) + print("ra, dec =", ra, dec) stars = agasc.get_agasc_cone( ra, dec, radius=radius, agasc_file=agasc_file, fix_color1=False ) - stars.sort('AGASC_ID') + stars.sort("AGASC_ID") - for agasc_id in stars['AGASC_ID'][:nstar_limit]: - print(' agasc_id =', agasc_id) + for agasc_id in stars["AGASC_ID"][:nstar_limit]: + print(" agasc_id =", agasc_id) star1 = agasc.get_star(agasc_id, agasc_file=agasc_file, fix_color1=False) star2 = mp_get_agascid(agasc_id) for colname in AGASC_COLNAMES: - if star1[colname].dtype.kind == 'f': + if star1[colname].dtype.kind == "f": assert np.all(np.allclose(star1[colname], star2[colname])) else: assert star1[colname] == star2[colname] @@ -396,84 +396,84 @@ def test_proseco_agasc_1p7(): # Stars looking toward galactic center (dense!) p_stars = agasc.get_agasc_cone( - -266, -29, 3, agasc_file=proseco_file, date='2000:001' + -266, -29, 3, agasc_file=proseco_file, date="2000:001" ) - m_stars = agasc.get_agasc_cone(-266, -29, 3, agasc_file=mini_file, date='2000:001') + m_stars = agasc.get_agasc_cone(-266, -29, 3, agasc_file=mini_file, date="2000:001") # Every miniagasc_1p7 star is in proseco_agasc_1p7 - m_ids = m_stars['AGASC_ID'] - p_ids = p_stars['AGASC_ID'] + m_ids = m_stars["AGASC_ID"] + p_ids = p_stars["AGASC_ID"] assert set(m_ids) < set(p_ids) # Values are exactly the same p_id_map = {p_ids[idx]: idx for idx in np.arange(len(p_ids))} for m_star in m_stars: - m_id = m_star['AGASC_ID'] + m_id = m_star["AGASC_ID"] p_star = p_stars[p_id_map[m_id]] for name in p_star.colnames: assert p_star[name] == m_star[name] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_agasc_cone(): ra, dec = 282.53, -0.38 # Obsid 22429 with a couple of color1=1.5 stars stars1 = agasc.get_agasc_cone( - ra, dec, date='2021:001', agasc_file=MINIAGASC, use_supplement=False + ra, dec, date="2021:001", agasc_file=MINIAGASC, use_supplement=False ) stars2 = agasc.get_agasc_cone( - ra, dec, date='2021:001', agasc_file=MINIAGASC, use_supplement=True + ra, dec, date="2021:001", agasc_file=MINIAGASC, use_supplement=True ) - ok = stars2['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT + ok = stars2["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT - change_names = ['MAG_CATID', 'COLOR1', 'MAG_ACA', 'MAG_ACA_ERR'] + change_names = ["MAG_CATID", "COLOR1", "MAG_ACA", "MAG_ACA_ERR"] for name in set(stars1.colnames) - set(change_names): assert np.all(stars1[name] == stars2[name]) - assert not np.any(stars1['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT) + assert not np.any(stars1["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT) # At least 35 stars in this field observed assert np.count_nonzero(ok) >= 35 # At least 7 color=1.5 stars converted to 1.49 (note: total of 23 color=1.5 # stars in this field) - assert np.count_nonzero(stars1['COLOR1'][ok] == 1.49) == 0 - assert np.count_nonzero(stars1['COLOR1'][ok] == 1.50) >= 7 - assert np.count_nonzero(stars2['COLOR1'][ok] == 1.49) >= 7 - assert np.count_nonzero(stars2['COLOR1'][ok] == 1.50) == 0 + assert np.count_nonzero(stars1["COLOR1"][ok] == 1.49) == 0 + assert np.count_nonzero(stars1["COLOR1"][ok] == 1.50) >= 7 + assert np.count_nonzero(stars2["COLOR1"][ok] == 1.49) >= 7 + assert np.count_nonzero(stars2["COLOR1"][ok] == 1.50) == 0 # For the stars that have updated data for the supplement, confirm they don't # have all the same values for MAG_ACA_ERR as the catalog values. Note this # is an integer column. - assert np.any(stars2['MAG_ACA_ERR'][ok] != stars1['MAG_ACA_ERR'][ok]) + assert np.any(stars2["MAG_ACA_ERR"][ok] != stars1["MAG_ACA_ERR"][ok]) # Similarly, in this set the stars with updated magnitudes are different from # the catalog values. - assert np.all(stars2['MAG_ACA'][ok] != stars1['MAG_ACA'][ok]) + assert np.all(stars2["MAG_ACA"][ok] != stars1["MAG_ACA"][ok]) - assert np.all(stars2['MAG_ACA_ERR'][~ok] == stars1['MAG_ACA_ERR'][~ok]) - assert np.all(stars2['MAG_ACA'][~ok] == stars1['MAG_ACA'][~ok]) + assert np.all(stars2["MAG_ACA_ERR"][~ok] == stars1["MAG_ACA_ERR"][~ok]) + assert np.all(stars2["MAG_ACA"][~ok] == stars1["MAG_ACA"][~ok]) -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_star(): agasc_id = 58720672 # Also checks that the default is False given the os.environ override for # this test file. star1 = agasc.get_star(agasc_id, agasc_file=MINIAGASC) star2 = agasc.get_star(agasc_id, agasc_file=MINIAGASC, use_supplement=True) - assert star1['MAG_CATID'] != agasc.MAG_CATID_SUPPLEMENT - assert star2['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT + assert star1["MAG_CATID"] != agasc.MAG_CATID_SUPPLEMENT + assert star2["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT - assert star1['AGASC_ID'] == star2['AGASC_ID'] + assert star1["AGASC_ID"] == star2["AGASC_ID"] - assert np.isclose(star1['COLOR1'], 1.50) - assert np.isclose(star2['COLOR1'], 1.49) + assert np.isclose(star1["COLOR1"], 1.50) + assert np.isclose(star2["COLOR1"], 1.49) - assert star2['MAG_ACA'] != star1['MAG_ACA'] - assert star2['MAG_ACA_ERR'] != star1['MAG_ACA_ERR'] + assert star2["MAG_ACA"] != star1["MAG_ACA"] + assert star2["MAG_ACA_ERR"] != star1["MAG_ACA_ERR"] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_star_disable_context_manager(): """Test that disable_supplement_mags context manager works. @@ -481,103 +481,103 @@ def test_supplement_get_star_disable_context_manager(): tests. """ agasc_id = 58720672 - star1 = agasc.get_star(agasc_id, date='2020:001', use_supplement=True) + star1 = agasc.get_star(agasc_id, date="2020:001", use_supplement=True) with agasc.set_supplement_enabled(True): - star2 = agasc.get_star(agasc_id, date='2020:001') + star2 = agasc.get_star(agasc_id, date="2020:001") for name in star1.colnames: assert star1[name] == star2[name] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") @agasc.set_supplement_enabled(True) def test_supplement_get_star_disable_decorator(): """Test that disable_supplement_mags context manager works""" agasc_id = 58720672 - star1 = agasc.get_star(agasc_id, date='2020:001') - star2 = agasc.get_star(agasc_id, date='2020:001', use_supplement=True) + star1 = agasc.get_star(agasc_id, date="2020:001") + star2 = agasc.get_star(agasc_id, date="2020:001", use_supplement=True) for name in star1.colnames: assert star1[name] == star2[name] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_stars(): agasc_ids = [58720672, 670303120] star1 = agasc.get_stars(agasc_ids, agasc_file=MINIAGASC) star2 = agasc.get_stars(agasc_ids, agasc_file=MINIAGASC, use_supplement=True) - assert np.all(star1['MAG_CATID'] != agasc.MAG_CATID_SUPPLEMENT) - assert np.all(star2['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT) + assert np.all(star1["MAG_CATID"] != agasc.MAG_CATID_SUPPLEMENT) + assert np.all(star2["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT) - assert np.all(star1['AGASC_ID'] == star2['AGASC_ID']) + assert np.all(star1["AGASC_ID"] == star2["AGASC_ID"]) - assert np.allclose(star1['COLOR1'], [1.5, 0.24395067]) - assert np.allclose(star2['COLOR1'], [1.49, 0.24395067]) + assert np.allclose(star1["COLOR1"], [1.5, 0.24395067]) + assert np.allclose(star2["COLOR1"], [1.49, 0.24395067]) - assert np.all(star2['MAG_ACA'] != star1['MAG_ACA']) + assert np.all(star2["MAG_ACA"] != star1["MAG_ACA"]) def test_get_supplement_table_bad(): - bad = agasc.get_supplement_table('bad') + bad = agasc.get_supplement_table("bad") assert isinstance(bad, Table) - assert bad.colnames == ['agasc_id', 'source'] + assert bad.colnames == ["agasc_id", "source"] assert len(bad) > 3300 - assert 797847184 in bad['agasc_id'] + assert 797847184 in bad["agasc_id"] def test_get_supplement_table_bad_dict(): - bad = agasc.get_supplement_table('bad', as_dict=True) + bad = agasc.get_supplement_table("bad", as_dict=True) assert isinstance(bad, dict) assert len(bad) > 3300 - assert bad[797847184] == {'source': 1} + assert bad[797847184] == {"source": 1} @agasc.set_supplement_enabled(True) def test_get_bad_star_with_supplement(): agasc_id = 797847184 star = agasc.get_star(agasc_id, use_supplement=True) - assert star['CLASS'] == agasc.BAD_CLASS_SUPPLEMENT + assert star["CLASS"] == agasc.BAD_CLASS_SUPPLEMENT def test_bad_agasc_supplement_env_var(): try: - os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = 'asdfasdf' - with pytest.raises(ValueError, match='env var must be either'): + os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = "asdfasdf" + with pytest.raises(ValueError, match="env var must be either"): agasc.get_star(797847184) finally: - os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = 'False' + os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = "False" -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_get_supplement_table_mags(): - mags = agasc.get_supplement_table('mags') + mags = agasc.get_supplement_table("mags") assert isinstance(mags, Table) - assert 131736 in mags['agasc_id'] + assert 131736 in mags["agasc_id"] assert len(mags) > 80000 - assert mags.colnames == ['agasc_id', 'mag_aca', 'mag_aca_err', 'last_obs_time'] + assert mags.colnames == ["agasc_id", "mag_aca", "mag_aca_err", "last_obs_time"] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_get_supplement_table_mags_dict(): - mags = agasc.get_supplement_table('mags', as_dict=True) + mags = agasc.get_supplement_table("mags", as_dict=True) assert isinstance(mags, dict) assert 131736 in mags assert len(mags) > 80000 - assert list(mags[131736].keys()) == ['mag_aca', 'mag_aca_err', 'last_obs_time'] + assert list(mags[131736].keys()) == ["mag_aca", "mag_aca_err", "last_obs_time"] -@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason='no obs in supplement') +@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason="no obs in supplement") def test_get_supplement_table_obs(): - obs = agasc.get_supplement_table('obs') + obs = agasc.get_supplement_table("obs") assert isinstance(obs, Table) assert obs.colnames == [ - 'mp_starcat_time', - 'agasc_id', - 'obsid', - 'status', - 'comments', + "mp_starcat_time", + "agasc_id", + "obsid", + "status", + "comments", ] -@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason='no obs in supplement') +@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason="no obs in supplement") def test_get_supplement_table_obs_dict(): - obs = agasc.get_supplement_table('obs', as_dict=True) + obs = agasc.get_supplement_table("obs", as_dict=True) assert isinstance(obs, dict) diff --git a/agasc/tests/test_obs_status.py b/agasc/tests/test_obs_status.py index 55282a11..6119eb77 100644 --- a/agasc/tests/test_obs_status.py +++ b/agasc/tests/test_obs_status.py @@ -11,7 +11,7 @@ from agasc.scripts import update_supplement from agasc.supplement.utils import OBS_DTYPE, BAD_DTYPE -TEST_DATA_DIR = pathlib.Path(__file__).parent / 'data' +TEST_DATA_DIR = pathlib.Path(__file__).parent / "data" # for the purposes of the supplement tables # star_obs_catalogs.STARS_OBS is used to determine all AGASC IDs in an observation @@ -19,93 +19,93 @@ # this array is used to monkey-patch star_obs_catalogs.STARS_OBS STARS_OBS = np.array( [ - (56314, 114950168, '2010:110:14:57:43.442'), - (56314, 114950584, '2010:110:14:57:43.442'), - (56314, 114952056, '2010:110:14:57:43.442'), - (56314, 114952792, '2010:110:14:57:43.442'), - (56314, 114952824, '2010:110:14:57:43.442'), - (56314, 114955056, '2010:110:14:57:43.442'), - (56314, 114956608, '2010:110:14:57:43.442'), - (56314, 115347520, '2010:110:14:57:43.442'), - (56312, 357045496, '2010:110:16:59:51.399'), - (56312, 357049064, '2010:110:16:59:51.399'), - (56312, 357051640, '2010:110:16:59:51.399'), - (56312, 357054680, '2010:110:16:59:51.399'), - (56312, 358220224, '2010:110:16:59:51.399'), - (56312, 358222256, '2010:110:16:59:51.399'), - (56312, 358224400, '2010:110:16:59:51.399'), - (56312, 358757768, '2010:110:16:59:51.399'), - (56313, 441853632, '2010:110:15:07:49.876'), - (56313, 441854760, '2010:110:15:07:49.876'), - (56313, 441855776, '2010:110:15:07:49.876'), - (56313, 441856032, '2010:110:15:07:49.876'), - (56313, 441856400, '2010:110:15:07:49.876'), - (56313, 441980072, '2010:110:15:07:49.876'), - (56313, 491391592, '2010:110:15:07:49.876'), - (56313, 491394504, '2010:110:15:07:49.876'), - (56311, 563087864, '2010:110:18:59:46.600'), - (56311, 563088952, '2010:110:18:59:46.600'), - (56311, 563089432, '2010:110:18:59:46.600'), - (56311, 563091784, '2010:110:18:59:46.600'), - (56311, 563092520, '2010:110:18:59:46.600'), - (56311, 563612488, '2010:110:18:59:46.600'), - (56311, 563612792, '2010:110:18:59:46.600'), - (56311, 563617352, '2010:110:18:59:46.600'), - (56310, 624826320, '2010:110:20:33:54.789'), - (56310, 624826464, '2010:110:20:33:54.789'), - (56310, 624828488, '2010:110:20:33:54.789'), - (56310, 624831328, '2010:110:20:33:54.789'), - (56310, 624831392, '2010:110:20:33:54.789'), - (56310, 624954248, '2010:110:20:33:54.789'), - (56310, 624956216, '2010:110:20:33:54.789'), - (56310, 625476960, '2010:110:20:33:54.789'), - (12203, 697581832, '2010:111:10:30:46.876'), - (12203, 697963056, '2010:111:10:30:46.876'), - (12203, 697963288, '2010:111:10:30:46.876'), - (12203, 697970576, '2010:111:10:30:46.876'), - (12203, 697973824, '2010:111:10:30:46.876'), - (56308, 732697144, '2010:110:23:23:49.708'), - (56308, 732698416, '2010:110:23:23:49.708'), - (56309, 762184312, '2010:110:22:02:30.780'), - (56309, 762184768, '2010:110:22:02:30.780'), - (56309, 762185584, '2010:110:22:02:30.780'), - (56309, 762186016, '2010:110:22:02:30.780'), - (56309, 762186080, '2010:110:22:02:30.780'), - (56309, 762191224, '2010:110:22:02:30.780'), - (56309, 762579584, '2010:110:22:02:30.780'), - (56309, 762581024, '2010:110:22:02:30.780'), - (56308, 806748432, '2010:110:23:23:49.708'), - (56308, 806748880, '2010:110:23:23:49.708'), - (56308, 806750112, '2010:110:23:23:49.708'), - (56308, 806750408, '2010:110:23:23:49.708'), - (56308, 806750912, '2010:110:23:23:49.708'), - (56308, 806751424, '2010:110:23:23:49.708'), - (56306, 956708808, '2010:111:02:18:49.052'), - (56306, 957219128, '2010:111:02:18:49.052'), - (56306, 957221200, '2010:111:02:18:49.052'), - (56306, 957222432, '2010:111:02:18:49.052'), - (56306, 957229080, '2010:111:02:18:49.052'), - (56306, 957230976, '2010:111:02:18:49.052'), - (56306, 957233920, '2010:111:02:18:49.052'), - (56306, 957369088, '2010:111:02:18:49.052'), - (11849, 1019347720, '2010:111:12:26:54.536'), - (11849, 1019348536, '2010:111:12:26:54.536'), - (11849, 1019350904, '2010:111:12:26:54.536'), - (11849, 1019354232, '2010:111:12:26:54.536'), - (11849, 1019357032, '2010:111:12:26:54.536'), - (11980, 1198184872, '2010:111:03:21:11.299'), - (11980, 1198190648, '2010:111:03:21:11.299'), - (11980, 1198190664, '2010:111:03:21:11.299'), - (11980, 1198191400, '2010:111:03:21:11.299'), - (11980, 1198192456, '2010:111:03:21:11.299'), + (56314, 114950168, "2010:110:14:57:43.442"), + (56314, 114950584, "2010:110:14:57:43.442"), + (56314, 114952056, "2010:110:14:57:43.442"), + (56314, 114952792, "2010:110:14:57:43.442"), + (56314, 114952824, "2010:110:14:57:43.442"), + (56314, 114955056, "2010:110:14:57:43.442"), + (56314, 114956608, "2010:110:14:57:43.442"), + (56314, 115347520, "2010:110:14:57:43.442"), + (56312, 357045496, "2010:110:16:59:51.399"), + (56312, 357049064, "2010:110:16:59:51.399"), + (56312, 357051640, "2010:110:16:59:51.399"), + (56312, 357054680, "2010:110:16:59:51.399"), + (56312, 358220224, "2010:110:16:59:51.399"), + (56312, 358222256, "2010:110:16:59:51.399"), + (56312, 358224400, "2010:110:16:59:51.399"), + (56312, 358757768, "2010:110:16:59:51.399"), + (56313, 441853632, "2010:110:15:07:49.876"), + (56313, 441854760, "2010:110:15:07:49.876"), + (56313, 441855776, "2010:110:15:07:49.876"), + (56313, 441856032, "2010:110:15:07:49.876"), + (56313, 441856400, "2010:110:15:07:49.876"), + (56313, 441980072, "2010:110:15:07:49.876"), + (56313, 491391592, "2010:110:15:07:49.876"), + (56313, 491394504, "2010:110:15:07:49.876"), + (56311, 563087864, "2010:110:18:59:46.600"), + (56311, 563088952, "2010:110:18:59:46.600"), + (56311, 563089432, "2010:110:18:59:46.600"), + (56311, 563091784, "2010:110:18:59:46.600"), + (56311, 563092520, "2010:110:18:59:46.600"), + (56311, 563612488, "2010:110:18:59:46.600"), + (56311, 563612792, "2010:110:18:59:46.600"), + (56311, 563617352, "2010:110:18:59:46.600"), + (56310, 624826320, "2010:110:20:33:54.789"), + (56310, 624826464, "2010:110:20:33:54.789"), + (56310, 624828488, "2010:110:20:33:54.789"), + (56310, 624831328, "2010:110:20:33:54.789"), + (56310, 624831392, "2010:110:20:33:54.789"), + (56310, 624954248, "2010:110:20:33:54.789"), + (56310, 624956216, "2010:110:20:33:54.789"), + (56310, 625476960, "2010:110:20:33:54.789"), + (12203, 697581832, "2010:111:10:30:46.876"), + (12203, 697963056, "2010:111:10:30:46.876"), + (12203, 697963288, "2010:111:10:30:46.876"), + (12203, 697970576, "2010:111:10:30:46.876"), + (12203, 697973824, "2010:111:10:30:46.876"), + (56308, 732697144, "2010:110:23:23:49.708"), + (56308, 732698416, "2010:110:23:23:49.708"), + (56309, 762184312, "2010:110:22:02:30.780"), + (56309, 762184768, "2010:110:22:02:30.780"), + (56309, 762185584, "2010:110:22:02:30.780"), + (56309, 762186016, "2010:110:22:02:30.780"), + (56309, 762186080, "2010:110:22:02:30.780"), + (56309, 762191224, "2010:110:22:02:30.780"), + (56309, 762579584, "2010:110:22:02:30.780"), + (56309, 762581024, "2010:110:22:02:30.780"), + (56308, 806748432, "2010:110:23:23:49.708"), + (56308, 806748880, "2010:110:23:23:49.708"), + (56308, 806750112, "2010:110:23:23:49.708"), + (56308, 806750408, "2010:110:23:23:49.708"), + (56308, 806750912, "2010:110:23:23:49.708"), + (56308, 806751424, "2010:110:23:23:49.708"), + (56306, 956708808, "2010:111:02:18:49.052"), + (56306, 957219128, "2010:111:02:18:49.052"), + (56306, 957221200, "2010:111:02:18:49.052"), + (56306, 957222432, "2010:111:02:18:49.052"), + (56306, 957229080, "2010:111:02:18:49.052"), + (56306, 957230976, "2010:111:02:18:49.052"), + (56306, 957233920, "2010:111:02:18:49.052"), + (56306, 957369088, "2010:111:02:18:49.052"), + (11849, 1019347720, "2010:111:12:26:54.536"), + (11849, 1019348536, "2010:111:12:26:54.536"), + (11849, 1019350904, "2010:111:12:26:54.536"), + (11849, 1019354232, "2010:111:12:26:54.536"), + (11849, 1019357032, "2010:111:12:26:54.536"), + (11980, 1198184872, "2010:111:03:21:11.299"), + (11980, 1198190648, "2010:111:03:21:11.299"), + (11980, 1198190664, "2010:111:03:21:11.299"), + (11980, 1198191400, "2010:111:03:21:11.299"), + (11980, 1198192456, "2010:111:03:21:11.299"), ], - dtype=[('obsid', ' 0, 'Table.write was never called' + assert len(mock_write.calls) > 0, "Table.write was never called" def test_update_obs(monkeypatch, mock_open): - monkeypatch.setattr(star_obs_catalogs, 'STARS_OBS', STARS_OBS) + monkeypatch.setattr(star_obs_catalogs, "STARS_OBS", STARS_OBS) def mock_write(*args, **kwargs): mock_write.calls.append((args, kwargs)) - if 'path' in kwargs and kwargs['path'] == 'bad': + if "path" in kwargs and kwargs["path"] == "bad": mock_write.n_calls += 1 ref = table.Table() assert False - if 'path' in kwargs and kwargs['path'] == 'obs': + if "path" in kwargs and kwargs["path"] == "obs": mock_write.n_calls += 1 ref = table.Table( np.array( [ - (56311, 563087864, 1, '', '2010:110:18:59:46.600'), - (56311, 563088952, 1, '', '2010:110:18:59:46.600'), - (56311, 563089432, 1, '', '2010:110:18:59:46.600'), - (56311, 563091784, 1, '', '2010:110:18:59:46.600'), - (56311, 563092520, 1, '', '2010:110:18:59:46.600'), - (56311, 563612488, 1, '', '2010:110:18:59:46.600'), - (56311, 563612792, 1, '', '2010:110:18:59:46.600'), - (56311, 563617352, 1, '', '2010:110:18:59:46.600'), - (56308, 806750112, 0, '', '2010:110:23:23:49.708'), + (56311, 563087864, 1, "", "2010:110:18:59:46.600"), + (56311, 563088952, 1, "", "2010:110:18:59:46.600"), + (56311, 563089432, 1, "", "2010:110:18:59:46.600"), + (56311, 563091784, 1, "", "2010:110:18:59:46.600"), + (56311, 563092520, 1, "", "2010:110:18:59:46.600"), + (56311, 563612488, 1, "", "2010:110:18:59:46.600"), + (56311, 563612792, 1, "", "2010:110:18:59:46.600"), + (56311, 563617352, 1, "", "2010:110:18:59:46.600"), + (56308, 806750112, 0, "", "2010:110:23:23:49.708"), ( 11849, 1019348536, 1, - 'just removed them', - '2010:111:12:26:54.536', + "just removed them", + "2010:111:12:26:54.536", ), ( 11849, 1019350904, 1, - 'just removed them', - '2010:111:12:26:54.536', + "just removed them", + "2010:111:12:26:54.536", ), ( 56314, 114950168, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 114950584, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 114952056, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 114952792, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 114952824, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 114955056, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 114956608, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ( 56314, 115347520, 1, - 'removed because I felt like it', - '2010:110:14:57:43.442', + "removed because I felt like it", + "2010:110:14:57:43.442", ), ], dtype=[ - ('obsid', ' 0, "Table.write was never called" def test_override(monkeypatch): _monkeypatch_star_obs_catalogs_( - monkeypatch, test_file=TEST_DATA_DIR / 'mag-stats.h5', path='/obs_status' + monkeypatch, test_file=TEST_DATA_DIR / "mag-stats.h5", path="/obs_status" ) telem = _monkeypatch_get_telemetry_( - monkeypatch, test_file=TEST_DATA_DIR / 'mag-stats.h5', path='/obs_status/telem' + monkeypatch, test_file=TEST_DATA_DIR / "mag-stats.h5", path="/obs_status/telem" ) # Case 1. There are two previously unknown suspect observations out of 5. @@ -1224,63 +1224,63 @@ def test_override(monkeypatch): assert len(fails) == 2 assert len(obs_stats) == 5 - assert agasc_stats['n_obsids_fail'] == 2 - assert agasc_stats['n_obsids_ok'] == 3 + assert agasc_stats["n_obsids_fail"] == 2 + assert agasc_stats["n_obsids_ok"] == 3 assert ( - agasc_stats['n_obsids_suspect'] == 2 + agasc_stats["n_obsids_suspect"] == 2 ) # two suspect count as "fail" in this context assert not np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800])]["mags"]), agasc_stats["mag_obs"] ) assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800, 23682, 23683])]['mags']), - agasc_stats['mag_obs'], + np.mean(telem[np.in1d(telem["obsid"], [12800, 23682, 23683])]["mags"]), + agasc_stats["mag_obs"], ) # Case 2. Four observations (including the suspect) are marked with non-zero status # Only the remaining observation should be included obs_status_override = { - ('2018:296:15:53:14.596', 10492752): {'status': 1, 'comments': ''}, - ('2021:015:00:01:45.585', 10492752): {'status': 1, 'comments': ''}, - ('2021:089:02:48:00.575', 10492752): {'status': 1, 'comments': ''}, - ('2021:201:02:58:03.250', 10492752): {'status': 1, 'comments': ''}, + ("2018:296:15:53:14.596", 10492752): {"status": 1, "comments": ""}, + ("2021:015:00:01:45.585", 10492752): {"status": 1, "comments": ""}, + ("2021:089:02:48:00.575", 10492752): {"status": 1, "comments": ""}, + ("2021:201:02:58:03.250", 10492752): {"status": 1, "comments": ""}, } agasc_stats, obs_stats, fails = mag_estimate.get_agasc_id_stats( 10492752, obs_status_override=obs_status_override ) assert len(fails) == 0 - assert agasc_stats['n_obsids_fail'] == 0 - assert agasc_stats['n_obsids_ok'] == 1 + assert agasc_stats["n_obsids_fail"] == 0 + assert agasc_stats["n_obsids_ok"] == 1 assert ( - agasc_stats['n_obsids_suspect'] == 0 + agasc_stats["n_obsids_suspect"] == 0 ) # no fails because all status==1 skipped assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800])]["mags"]), agasc_stats["mag_obs"] ) assert not np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800, 23682, 23683])]['mags']), - agasc_stats['mag_obs'], + np.mean(telem[np.in1d(telem["obsid"], [12800, 23682, 23683])]["mags"]), + agasc_stats["mag_obs"], ) # Case 3: # - One of the suspect observations is previously known and marked as OK (status=0) # - One other observation is marked as not-OK obs_status_override = { - ('2021:015:00:01:45.585', 10492752): {'status': 0, 'comments': ''}, - ('2021:089:02:48:00.575', 10492752): {'status': 1, 'comments': ''}, + ("2021:015:00:01:45.585", 10492752): {"status": 0, "comments": ""}, + ("2021:089:02:48:00.575", 10492752): {"status": 1, "comments": ""}, } agasc_stats, obs_stats, fails = mag_estimate.get_agasc_id_stats( 10492752, obs_status_override=obs_status_override ) assert len(fails) == 1 - assert agasc_stats['n_obsids_fail'] == 1 - assert agasc_stats['n_obsids_ok'] == 3 - assert agasc_stats['n_obsids_suspect'] == 1 # one failed + assert agasc_stats["n_obsids_fail"] == 1 + assert agasc_stats["n_obsids_ok"] == 3 + assert agasc_stats["n_obsids_suspect"] == 1 # one failed assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800, 23681, 23683])]['mags']), - agasc_stats['mag_obs'], + np.mean(telem[np.in1d(telem["obsid"], [12800, 23681, 23683])]["mags"]), + agasc_stats["mag_obs"], ) @@ -1298,14 +1298,14 @@ def _remove_list_duplicates(a_list): def _monkeypatch_star_obs_catalogs_(monkeypatch, test_file, path): tables = [ - 'STARS_OBS', + "STARS_OBS", ] - res = {t: table.Table.read(test_file, path=f'{path}/cat/{t}') for t in tables} + res = {t: table.Table.read(test_file, path=f"{path}/cat/{t}") for t in tables} for k in res: res[k].convert_bytestring_to_unicode() - res['STARS_OBS'].add_index('agasc_id') - res['STARS_OBS'].add_index('mp_starcat_time') + res["STARS_OBS"].add_index("agasc_id") + res["STARS_OBS"].add_index("mp_starcat_time") for k in res: monkeypatch.setattr(star_obs_catalogs, k, res[k]) @@ -1315,17 +1315,17 @@ def _monkeypatch_get_telemetry_(monkeypatch, test_file, path): telem = table.Table.read(test_file, path=path) def get_telemetry(obs): - obsid = obs['obsid'] - if obs['obsid'] in telem['obsid']: - return telem[telem['obsid'] == obs['obsid']] - raise Exception(f'{obsid=} not in test telemetry') + obsid = obs["obsid"] + if obs["obsid"] in telem["obsid"]: + return telem[telem["obsid"] == obs["obsid"]] + raise Exception(f"{obsid=} not in test telemetry") - monkeypatch.setattr(mag_estimate, 'get_telemetry', get_telemetry) + monkeypatch.setattr(mag_estimate, "get_telemetry", get_telemetry) return telem -def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): +def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / "mag-stats.h5"): """ Create data to test mag-stats. @@ -1343,20 +1343,20 @@ def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): star_obs_catalogs.load() mp_starcat_time = [ - '2011:288:06:14:49.501', - '2021:015:00:01:45.585', - '2021:089:02:48:00.575', - '2021:201:02:58:03.250', - '2018:296:15:53:14.596', + "2011:288:06:14:49.501", + "2021:015:00:01:45.585", + "2021:089:02:48:00.575", + "2021:201:02:58:03.250", + "2018:296:15:53:14.596", ] STARS_OBS = star_obs_catalogs.STARS_OBS[ - np.in1d(star_obs_catalogs.STARS_OBS['mp_starcat_time'], mp_starcat_time) + np.in1d(star_obs_catalogs.STARS_OBS["mp_starcat_time"], mp_starcat_time) ] - STARS_OBS = STARS_OBS.group_by('agasc_id') - STARS_OBS.add_index('agasc_id') + STARS_OBS = STARS_OBS.group_by("agasc_id") + STARS_OBS.add_index("agasc_id") STARS_OBS.write( filename, - path='/obs_status/cat/STARS_OBS', + path="/obs_status/cat/STARS_OBS", serialize_meta=True, append=True, overwrite=True, @@ -1368,44 +1368,44 @@ def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): # might be in maneuver mode or in acquisition. These times come from the kadi events v1 version, # but they do not matter much. telem_by_obsid = [ - telem[(telem['obsid'] == 12800) & (telem['times'] > 435047672.0)][:100], + telem[(telem["obsid"] == 12800) & (telem["times"] > 435047672.0)][:100], # only 10 points, excluding the beginning - telem[(telem['obsid'] == 23681) & (telem['times'] > 727057549.0)][:10], - telem[(telem['obsid'] == 23682) & (telem['times'] > 733462165.0)][:100], - telem[(telem['obsid'] == 23683) & (telem['times'] > 743139160.0)][:100], - telem[(telem['obsid'] == 48900) & (telem['times'] > 656698074.0)][:100], + telem[(telem["obsid"] == 23681) & (telem["times"] > 727057549.0)][:10], + telem[(telem["obsid"] == 23682) & (telem["times"] > 733462165.0)][:100], + telem[(telem["obsid"] == 23683) & (telem["times"] > 743139160.0)][:100], + telem[(telem["obsid"] == 48900) & (telem["times"] > 656698074.0)][:100], ] - telem_by_obsid[-1]['mags_img'] += 0.01 * np.exp(np.arange(100) / 20) - telem_by_obsid[-1]['mags'] += 0.01 * np.exp(np.arange(100) / 20) + telem_by_obsid[-1]["mags_img"] += 0.01 * np.exp(np.arange(100) / 20) + telem_by_obsid[-1]["mags"] += 0.01 * np.exp(np.arange(100) / 20) t = vstack(telem_by_obsid) - t.write(filename, path='/obs_status/telem', serialize_meta=True, append=True) + t.write(filename, path="/obs_status/telem", serialize_meta=True, append=True) -def recreate_test_supplement(supplement_filename=TEST_DATA_DIR / 'agasc_supplement.h5'): +def recreate_test_supplement(supplement_filename=TEST_DATA_DIR / "agasc_supplement.h5"): # this is not a test function, but a function to generate the test supplement from scratch # whenever it needs updating, so all the data is actually contained in this file from _pytest.monkeypatch import MonkeyPatch monkeypatch = MonkeyPatch() - monkeypatch.setitem(__builtins__, 'open', _open) - monkeypatch.setattr(star_obs_catalogs, 'STARS_OBS', STARS_OBS) + monkeypatch.setitem(__builtins__, "open", _open) + monkeypatch.setattr(star_obs_catalogs, "STARS_OBS", STARS_OBS) - with tables.open_file(str(supplement_filename), 'w'): + with tables.open_file(str(supplement_filename), "w"): pass - print(f'Updating {supplement_filename}') - status = update_supplement.parse_args(filename='file_0.yml') - print('obs') - print(status['obs']) + print(f"Updating {supplement_filename}") + status = update_supplement.parse_args(filename="file_0.yml") + print("obs") + print(status["obs"]) update_supplement.update_obs_table( - supplement_filename, status['obs'], dry_run=False + supplement_filename, status["obs"], dry_run=False ) - print('mags') - print(status['mags']) + print("mags") + print(status["mags"]) update_supplement.update_mags_table( - supplement_filename, status['mags'], dry_run=False + supplement_filename, status["mags"], dry_run=False ) - print('bad') - print(status['bad']) - update_supplement.add_bad_star(supplement_filename, status['bad'], dry_run=False) + print("bad") + print(status["bad"]) + update_supplement.add_bad_star(supplement_filename, status["bad"], dry_run=False) diff --git a/scripts/make_fits.py b/scripts/make_fits.py index a4057238..2437d9c7 100644 --- a/scripts/make_fits.py +++ b/scripts/make_fits.py @@ -7,25 +7,25 @@ NEW_COMMENT = ( - ' ', - ' THE AXAF GUIDE and ACQUISITION STAR CATALOG V1.7', - ' ', - ' An all-sky astrometric and photometric catalog', - ' prepared for the operation of AXAF.', - ' ', - 'This file contains data for one of the 9537 regions constituting', - 'the AXAF Guide and Acquisition Star Catalog (AGASC) version 1.7', - 'Additional information on the AGASC may be obtained from the', - 'AXAF Science Center, Smithsonian Astrophysical Observatory,', - '60 Garden St., Cambridge, MA 02138, or in tables elsewhere', + " ", + " THE AXAF GUIDE and ACQUISITION STAR CATALOG V1.7", + " ", + " An all-sky astrometric and photometric catalog", + " prepared for the operation of AXAF.", + " ", + "This file contains data for one of the 9537 regions constituting", + "the AXAF Guide and Acquisition Star Catalog (AGASC) version 1.7", + "Additional information on the AGASC may be obtained from the", + "AXAF Science Center, Smithsonian Astrophysical Observatory,", + "60 Garden St., Cambridge, MA 02138, or in tables elsewhere", 'in this set of volumes (see the "comments" file).', - 'Null values are listed as -9999.', - ' ', - 'V1.7 changes values for MAG_ACA, MAG_ACA_ERR, RSV1, RSV2 and RSV3', - ' ', - ' ', - 'File written on Sept 5 2018', - ' ', + "Null values are listed as -9999.", + " ", + "V1.7 changes values for MAG_ACA, MAG_ACA_ERR, RSV1, RSV2 and RSV3", + " ", + " ", + "File written on Sept 5 2018", + " ", ) @@ -33,7 +33,7 @@ def get_options(): parser = argparse.ArgumentParser( description="Make new AGASC fits files from old AGASC and new h5 source" ) - parser.add_argument("--h5", default='/proj/sot/ska/data/agasc/agasc1p7.h5') + parser.add_argument("--h5", default="/proj/sot/ska/data/agasc/agasc1p7.h5") parser.add_argument("--olddir", default="/data/agasc1p6/agasc") parser.add_argument("--out", default="./testagasc") args = parser.parse_args() @@ -45,30 +45,30 @@ def update_file(src, dest, newdata): hdu = fits.open(src) hdr = hdu[0].header # Update AGASC VERSION - hdr['EXTVER'] = 17 + hdr["EXTVER"] = 17 for row in hdu[1].data: # Set RSV1 to be -9999.0 by default (for most stars this is a no-op, already -9999.0) - row['RSV1'] = -9999.0 + row["RSV1"] = -9999.0 # Find the index of the AGASC ID we have in the sorted table of newdata - idx = np.searchsorted(newdata['AGASC_ID'], row['AGASC_ID']) - if newdata['AGASC_ID'][idx] != row['AGASC_ID']: - idx = np.flatnonzero(newdata['AGASC_ID'] == row['AGASC_ID'])[0] + idx = np.searchsorted(newdata["AGASC_ID"], row["AGASC_ID"]) + if newdata["AGASC_ID"][idx] != row["AGASC_ID"]: + idx = np.flatnonzero(newdata["AGASC_ID"] == row["AGASC_ID"])[0] nrow = newdata[idx] - if nrow['RSV3'] != 1.0: + if nrow["RSV3"] != 1.0: continue - row['MAG_ACA'] = nrow['MAG_ACA'] - row['MAG_ACA_ERR'] = nrow['MAG_ACA_ERR'] - row['RSV1'] = nrow['RSV1'] - row['RSV2'] = nrow['RSV2'] - row['RSV3'] = nrow['RSV3'] + row["MAG_ACA"] = nrow["MAG_ACA"] + row["MAG_ACA_ERR"] = nrow["MAG_ACA_ERR"] + row["RSV1"] = nrow["RSV1"] + row["RSV2"] = nrow["RSV2"] + row["RSV3"] = nrow["RSV3"] # Set all previous lines to a colon (replacing with a space doesn't # seem to actually work) - hdr['comment'][0:19] = ":" + hdr["comment"][0:19] = ":" for idx, line in enumerate(NEW_COMMENT): - hdr['comment'][idx] = line + hdr["comment"][idx] = line # Update table comments for RSV1 RSV2 RSV3 hdu[1].header.comments["TTYPE26"] = "APASS V - i magnitude (COLOR3)" @@ -82,9 +82,9 @@ def update_file(src, dest, newdata): def main(h5file, srcdir, outdir): # This sorts the full agasc in memory, so don't do it on a puny machine tbl = Table.read(h5file) - tbl.sort('AGASC_ID') + tbl.sort("AGASC_ID") # redefine outdir as the agasc subdirectory in outdir - outdir = os.path.join(outdir, 'agasc') + outdir = os.path.join(outdir, "agasc") if not os.path.exists(outdir): os.makedirs(outdir) for topdir in Path(srcdir).glob("*"): @@ -96,6 +96,6 @@ def main(h5file, srcdir, outdir): update_file(f, newfile, tbl) -if __name__ == '__main__': +if __name__ == "__main__": args = get_options() main(args.h5, args.olddir, args.out) diff --git a/scripts/make_links.py b/scripts/make_links.py index 36722a96..61de2029 100644 --- a/scripts/make_links.py +++ b/scripts/make_links.py @@ -3,7 +3,7 @@ from Ska.Shell import bash -for agasc_dir in Path('agasc').glob("*"): +for agasc_dir in Path("agasc").glob("*"): updir = agasc_dir.name.upper() if not os.path.exists(updir): os.makedirs(updir) diff --git a/setup.py b/setup.py index ab60e64a..bd1ddbdc 100644 --- a/setup.py +++ b/setup.py @@ -9,48 +9,48 @@ entry_points = { - 'console_scripts': [ - 'agasc-magnitudes-report=agasc.scripts.mag_estimate_report:main', - 'agasc-update-magnitudes=agasc.scripts.update_mag_supplement:main', - 'agasc-update-supplement=agasc.scripts.update_supplement:main', - 'agasc-supplement-tasks=agasc.scripts.supplement_tasks:main', - 'agasc-supplement-diff=agasc.scripts.supplement_diff:main', + "console_scripts": [ + "agasc-magnitudes-report=agasc.scripts.mag_estimate_report:main", + "agasc-update-magnitudes=agasc.scripts.update_mag_supplement:main", + "agasc-update-supplement=agasc.scripts.update_supplement:main", + "agasc-supplement-tasks=agasc.scripts.supplement_tasks:main", + "agasc-supplement-diff=agasc.scripts.supplement_diff:main", ] } data_files = [ ( - os.path.join('share', 'agasc'), + os.path.join("share", "agasc"), [ - 'task_schedules/task_schedule_supplement_dispositions.cfg', - 'task_schedules/task_schedule_update_supplement_rc.cfg', + "task_schedules/task_schedule_supplement_dispositions.cfg", + "task_schedules/task_schedule_update_supplement_rc.cfg", ], ) ] setup( - name='agasc', + name="agasc", use_scm_version=True, - setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], - description='AGASC catalog access', - author='Jean Connelly, Tom Aldcroft', - author_email='taldcroft@cfa.harvard.edu', - url='http://cxc.harvard.edu/mta/ASPECT/tool_doc/agasc', + setup_requires=["setuptools_scm", "setuptools_scm_git_archive"], + description="AGASC catalog access", + author="Jean Connelly, Tom Aldcroft", + author_email="taldcroft@cfa.harvard.edu", + url="http://cxc.harvard.edu/mta/ASPECT/tool_doc/agasc", packages=[ - 'agasc', - 'agasc.supplement', - 'agasc.supplement.magnitudes', - 'agasc.tests', - 'agasc.scripts', + "agasc", + "agasc.supplement", + "agasc.supplement.magnitudes", + "agasc.tests", + "agasc.scripts", ], package_data={ - 'agasc.supplement.magnitudes': ['templates/*'], - 'agasc.tests': ['data/*'], + "agasc.supplement.magnitudes": ["templates/*"], + "agasc.tests": ["data/*"], }, data_files=data_files, - tests_require=['pytest'], + tests_require=["pytest"], cmdclass=cmdclass, entry_points=entry_points, ) From 4b0d3484c575be6f79de28826561e5a63c41239b Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:50:12 -0400 Subject: [PATCH 06/10] Deal with Py2 files --- create_agasc_h5.py => create_agasc_h5.py2 | 0 ...agasc_h5.py => create_indexed_agasc_h5.py2 | 0 time_agasc.py | 26 ------------------- validate_agasc1p7.py => validate_agasc1p7.py2 | 0 4 files changed, 26 deletions(-) rename create_agasc_h5.py => create_agasc_h5.py2 (100%) rename create_indexed_agasc_h5.py => create_indexed_agasc_h5.py2 (100%) delete mode 100644 time_agasc.py rename validate_agasc1p7.py => validate_agasc1p7.py2 (100%) diff --git a/create_agasc_h5.py b/create_agasc_h5.py2 similarity index 100% rename from create_agasc_h5.py rename to create_agasc_h5.py2 diff --git a/create_indexed_agasc_h5.py b/create_indexed_agasc_h5.py2 similarity index 100% rename from create_indexed_agasc_h5.py rename to create_indexed_agasc_h5.py2 diff --git a/time_agasc.py b/time_agasc.py deleted file mode 100644 index 18c0edf2..00000000 --- a/time_agasc.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -import sys -from itertools import count -import time -import numpy as np -import agasc - - -def random_ra_dec(nsample): - x = np.random.uniform(-0.98, 0.98, size=nsample) - ras = 360 * np.random.random(nsample) - decs = np.degrees(np.arcsin(x)) - return ras, decs - -radius = 2.0 -nsample = 200 -ras, decs = random_ra_dec(nsample) - -print 'get_agasc_cone' -t0 = time.time() -for ra, dec, cnt in zip(ras, decs, count()): - x = agasc.get_agasc_cone(ra, dec, radius=radius, agasc_file='miniagasc.h5') - print cnt, len(ras), '\r', - sys.stdout.flush() -print -print time.time() - t0 diff --git a/validate_agasc1p7.py b/validate_agasc1p7.py2 similarity index 100% rename from validate_agasc1p7.py rename to validate_agasc1p7.py2 From dd8d64961794af117672d6c591e9bb3180f3efb3 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 20:51:23 -0400 Subject: [PATCH 07/10] Fix single quotes --- create_derived_agasc_h5.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/create_derived_agasc_h5.py b/create_derived_agasc_h5.py index 449c87ec..e721fcfe 100644 --- a/create_derived_agasc_h5.py +++ b/create_derived_agasc_h5.py @@ -167,16 +167,16 @@ def write_derived_agasc(filename: str, stars: np.ndarray, version_num: str): def filter_proseco_columns(stars): print("Excluding columns not needed for proseco") # fmt: off - excludes = ['PLX', 'PLX_ERR', 'PLX_CATID', - 'ACQQ1', 'ACQQ2', 'ACQQ3', 'ACQQ4', 'ACQQ5', 'ACQQ6', - 'XREF_ID1', 'XREF_ID2', 'XREF_ID3', 'XREF_ID4', 'XREF_ID5', - 'RSV4', 'RSV5', 'RSV6', - 'POS_CATID', 'PM_CATID', - 'MAG', 'MAG_ERR', 'MAG_BAND', 'MAG_CATID', - 'COLOR1_ERR', 'C1_CATID', # Keep color1, 2, 3 - 'COLOR2_ERR', 'C2_CATID', - 'RSV2', - 'VAR_CATID'] + excludes = ["PLX", "PLX_ERR", "PLX_CATID", + "ACQQ1", "ACQQ2", "ACQQ3", "ACQQ4", "ACQQ5", "ACQQ6", + "XREF_ID1", "XREF_ID2", "XREF_ID3", "XREF_ID4", "XREF_ID5", + "RSV4", "RSV5", "RSV6", + "POS_CATID", "PM_CATID", + "MAG", "MAG_ERR", "MAG_BAND", "MAG_CATID", + "COLOR1_ERR", "C1_CATID", # Keep color1, 2, 3 + "COLOR2_ERR", "C2_CATID", + "RSV2", + "VAR_CATID"] # fmt: on names = [name for name in stars.dtype.names if name not in excludes] From 5be35d96f0a9eabbae75656adb2230161474d611 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 21:04:18 -0400 Subject: [PATCH 08/10] ruff auto-fix --- agasc/agasc.py | 10 +-- agasc/healpix.py | 3 +- agasc/scripts/mag_estimate_report.py | 6 +- agasc/scripts/supplement_diff.py | 15 ++--- agasc/scripts/supplement_tasks.py | 9 ++- agasc/scripts/update_mag_supplement.py | 13 ++-- agasc/scripts/update_supplement.py | 12 ++-- agasc/supplement/magnitudes/mag_estimate.py | 25 ++++--- .../magnitudes/mag_estimate_report.py | 22 +++--- .../magnitudes/star_obs_catalogs.py | 6 +- .../magnitudes/update_mag_supplement.py | 67 ++++++++++--------- agasc/supplement/utils.py | 11 ++- agasc/tests/test_agasc_2.py | 6 +- agasc/tests/test_agasc_healpix.py | 1 - agasc/tests/test_obs_status.py | 19 +++--- create_derived_agasc_h5.py | 2 +- dev/profile_get_functions.py | 8 ++- dev/profile_memory.py | 24 +++---- dev/profile_memory_psrecord.py | 14 ---- scripts/make_fits.py | 6 +- scripts/make_links.py | 2 - setup.py | 3 +- 22 files changed, 135 insertions(+), 149 deletions(-) diff --git a/agasc/agasc.py b/agasc/agasc.py index 9573366a..0337a22e 100644 --- a/agasc/agasc.py +++ b/agasc/agasc.py @@ -3,7 +3,6 @@ import functools import os import re -from packaging.version import Version from pathlib import Path from typing import Optional @@ -12,6 +11,7 @@ import tables from astropy.table import Column, Table from Chandra.Time import DateTime +from packaging.version import Version from .healpix import get_stars_from_healpix_h5, is_healpix from .paths import default_agasc_dir @@ -367,8 +367,8 @@ def sphere_dist(ra1, dec1, ra2, dec2): dec1 = np.radians(dec1).astype(np.float64) dec2 = np.radians(dec2).astype(np.float64) - numerator = numexpr.evaluate( - "sin((dec2 - dec1) / 2) ** 2 + " # noqa + numerator = numexpr.evaluate( # noqa: F841 + "sin((dec2 - dec1) / 2) ** 2 + " "cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2" ) @@ -623,7 +623,7 @@ def _get_rows_read_where(ids_1d, dates_1d, agasc_file): rows = [] with tables.open_file(agasc_file) as h5: tbl = h5.root.data - for id, date in zip(ids_1d, dates_1d): + for id, _date in zip(ids_1d, dates_1d): id_rows = tbl.read_where("(AGASC_ID == {})".format(id)) if len(id_rows) > 1: @@ -645,7 +645,7 @@ def _get_rows_read_entire(ids_1d, dates_1d, agasc_file): agasc_idx = {agasc_id: idx for idx, agasc_id in enumerate(tbl["AGASC_ID"])} rows = [] - for agasc_id, date in zip(ids_1d, dates_1d): + for agasc_id, _date in zip(ids_1d, dates_1d): if agasc_id not in agasc_idx: raise IdNotFound(f"No entry found for {agasc_id} in AGASC") diff --git a/agasc/healpix.py b/agasc/healpix.py index cae876a4..68a5722f 100644 --- a/agasc/healpix.py +++ b/agasc/healpix.py @@ -19,7 +19,6 @@ import tables from astropy.table import Table - __all__ = ["is_healpix", "get_stars_from_healpix_h5"] @@ -106,7 +105,7 @@ def get_stars_from_healpix_h5( stars : astropy.table.Table Table of stars within the search cone, with columns from the AGASC data table. """ - from agasc import sphere_dist, read_h5_table + from agasc import read_h5_table, sphere_dist # Table of healpix, idx0, idx1 where idx is the index into main AGASC data table healpix_index_map, nside = get_healpix_info(agasc_file) diff --git a/agasc/scripts/mag_estimate_report.py b/agasc/scripts/mag_estimate_report.py index 9bd180fa..070a862a 100755 --- a/agasc/scripts/mag_estimate_report.py +++ b/agasc/scripts/mag_estimate_report.py @@ -4,11 +4,13 @@ Produce reports of the magnitude supplement. """ -import os import argparse +import os from pathlib import Path + import numpy as np -from astropy import table, time, units as u +from astropy import table, time +from astropy import units as u from cxotime import CxoTime, units from agasc.supplement.magnitudes import mag_estimate_report diff --git a/agasc/scripts/supplement_diff.py b/agasc/scripts/supplement_diff.py index 9d822459..ec21969f 100755 --- a/agasc/scripts/supplement_diff.py +++ b/agasc/scripts/supplement_diff.py @@ -4,19 +4,18 @@ Generate diff between to supplement files and output in HTML format. """ -import os -import tables -from pathlib import Path -import difflib -import pygments.lexers -import pygments.formatters import argparse import datetime - +import difflib +import os from io import StringIO +from pathlib import Path -from astropy.io import ascii +import pygments.formatters +import pygments.lexers +import tables from astropy import table +from astropy.io import ascii def read_file(filename, exclude=[]): diff --git a/agasc/scripts/supplement_tasks.py b/agasc/scripts/supplement_tasks.py index c7fc9dc8..1010f11c 100644 --- a/agasc/scripts/supplement_tasks.py +++ b/agasc/scripts/supplement_tasks.py @@ -9,18 +9,17 @@ * schedule-promotion: schedule supplement promotion """ -import os -import subprocess import argparse -from pathlib import Path -import shutil import getpass +import os import platform +import shutil +import subprocess from email.mime.text import MIMEText +from pathlib import Path from cxotime import CxoTime - AGASC_DATA = Path(os.environ["SKA"]) / "data" / "agasc" diff --git a/agasc/scripts/update_mag_supplement.py b/agasc/scripts/update_mag_supplement.py index 1393e5b4..6e4a2df8 100755 --- a/agasc/scripts/update_mag_supplement.py +++ b/agasc/scripts/update_mag_supplement.py @@ -5,17 +5,18 @@ """ -import os -from pathlib import Path import argparse import logging -import yaml +import os +from pathlib import Path + import pyyaks.logger +import yaml +from cxotime import CxoTime +from cxotime import units as u -from agasc.supplement.magnitudes import star_obs_catalogs -from agasc.supplement.magnitudes import update_mag_supplement from agasc.scripts import update_supplement -from cxotime import CxoTime, units as u +from agasc.supplement.magnitudes import star_obs_catalogs, update_mag_supplement def get_parser(): diff --git a/agasc/scripts/update_supplement.py b/agasc/scripts/update_supplement.py index 308b6ff8..12d8d02b 100755 --- a/agasc/scripts/update_supplement.py +++ b/agasc/scripts/update_supplement.py @@ -10,16 +10,16 @@ """ import argparse -from pathlib import Path import logging -from cxotime.cxotime import CxoTime -import yaml +from pathlib import Path + import numpy as np import pyyaks.logger +import yaml +from cxotime.cxotime import CxoTime from agasc.supplement.magnitudes import star_obs_catalogs as cat -from agasc.supplement.utils import update_mags_table, update_obs_table, add_bad_star - +from agasc.supplement.utils import add_bad_star, update_mags_table, update_obs_table logger = logging.getLogger("agasc.supplement") @@ -79,7 +79,7 @@ def _sanitize_args(status): ) if "agasc_id" not in value: - value["agasc_id"] = list(sorted(rows["agasc_id"])) + value["agasc_id"] = sorted(rows["agasc_id"]) else: value["agasc_id"] = list(np.atleast_1d(value["agasc_id"])) if "comments" not in value: diff --git a/agasc/supplement/magnitudes/mag_estimate.py b/agasc/supplement/magnitudes/mag_estimate.py index f808a193..c32dff96 100644 --- a/agasc/supplement/magnitudes/mag_estimate.py +++ b/agasc/supplement/magnitudes/mag_estimate.py @@ -2,30 +2,29 @@ Functions to estimate observed ACA magnitudes """ +import collections +import logging import sys import traceback -import logging -import collections -import scipy.stats -import scipy.special -import numpy as np import numba +import numpy as np +import scipy.special +import scipy.stats +import Ska.quatutil from astropy.table import Table, vstack - from Chandra.Time import DateTime -from cheta import fetch -from Quaternion import Quat -import Ska.quatutil -from mica.archive import aca_l0 -from mica.archive.aca_dark.dark_cal import get_dark_cal_image from chandra_aca.transform import count_rate_to_mag, pixels_to_yagzag +from cheta import fetch from cxotime import CxoTime from kadi import events +from mica.archive import aca_l0 +from mica.archive.aca_dark.dark_cal import get_dark_cal_image +from Quaternion import Quat -from . import star_obs_catalogs from agasc import get_star +from . import star_obs_catalogs logger = logging.getLogger("agasc.supplement") @@ -492,7 +491,7 @@ def get_telemetry_by_agasc_id(agasc_id, obsid=None, ignore_exceptions=False): if len(obs) > 1: obs = obs.loc["mp_starcat_time", sorted(obs["mp_starcat_time"])] telem = [] - for i, o in enumerate(obs): + for _i, o in enumerate(obs): try: t = Table(get_telemetry(o)) t["obsid"] = o["obsid"] diff --git a/agasc/supplement/magnitudes/mag_estimate_report.py b/agasc/supplement/magnitudes/mag_estimate_report.py index c4e88bf4..702ee6e5 100644 --- a/agasc/supplement/magnitudes/mag_estimate_report.py +++ b/agasc/supplement/magnitudes/mag_estimate_report.py @@ -1,26 +1,26 @@ -import platform +import copy +import errno import getpass +import json import logging -import errno import os -import copy -import json +import platform import warnings -from subprocess import Popen, PIPE -from pathlib import Path from email.mime.text import MIMEText +from pathlib import Path +from subprocess import PIPE, Popen + import jinja2 -import numpy as np import matplotlib.pyplot as plt +import numpy as np +from astropy import table +from cxotime import CxoTime from matplotlib.ticker import FixedLocator from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable from tqdm import tqdm -from astropy import table -from cxotime import CxoTime from agasc.supplement.magnitudes import mag_estimate - JINJA2 = jinja2.Environment( loader=jinja2.PackageLoader("agasc.supplement.magnitudes", "templates"), autoescape=jinja2.select_autoescape(["html", "xml"]), @@ -831,7 +831,7 @@ def plot_flags(telemetry, ax=None, obsid=None): ok = [f[1] for f in flags] labels = [f[0] for f in flags] - ticks = [i for i in range(len(flags))] + ticks = list(range(len(flags))) for i in range(len(ok)): ax.scatter( diff --git a/agasc/supplement/magnitudes/star_obs_catalogs.py b/agasc/supplement/magnitudes/star_obs_catalogs.py index b2808a73..82d89329 100644 --- a/agasc/supplement/magnitudes/star_obs_catalogs.py +++ b/agasc/supplement/magnitudes/star_obs_catalogs.py @@ -1,14 +1,12 @@ import os + import numpy as np import tables - -from astropy.table import Table, join from astropy import table - +from astropy.table import Table, join from chandra_aca.transform import yagzag_to_pixels from kadi import commands, events - STARS_OBS = None """The table of star observations""" diff --git a/agasc/supplement/magnitudes/update_mag_supplement.py b/agasc/supplement/magnitudes/update_mag_supplement.py index d4319a0c..481121cf 100755 --- a/agasc/supplement/magnitudes/update_mag_supplement.py +++ b/agasc/supplement/magnitudes/update_mag_supplement.py @@ -1,30 +1,31 @@ #!/usr/bin/env python -import traceback -import sys -import warnings -import os -import pickle import datetime import logging +import os +import pickle +import sys +import traceback +import warnings from functools import partial from multiprocessing import Pool -import jinja2 -from tqdm import tqdm -import tables +import jinja2 import numpy as np -from astropy import table -from astropy import time, units as u - +import tables +from astropy import table, time +from astropy import units as u +from cxotime import CxoTime from mica.starcheck import get_starcheck_catalog +from tqdm import tqdm + from agasc.supplement.magnitudes import ( - star_obs_catalogs, mag_estimate, + star_obs_catalogs, +) +from agasc.supplement.magnitudes import ( mag_estimate_report as msr, ) -from agasc.supplement.utils import save_version, MAGS_DTYPE -from cxotime import CxoTime - +from agasc.supplement.utils import MAGS_DTYPE, save_version logger = logging.getLogger("agasc.supplement") @@ -35,8 +36,8 @@ def level0_archive_time_range(): :return: tuple of CxoTime """ - import sqlite3 import os + import sqlite3 db_file = os.path.expandvars("$SKA/data/mica/archive/aca0/archfiles.db3") with sqlite3.connect(db_file) as connection: @@ -62,9 +63,10 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres :return: astropy.table.Table, astropy.table.Table, list obs_stats, agasc_stats, fails """ - from agasc.supplement.magnitudes import mag_estimate from astropy.table import Table, vstack + from agasc.supplement.magnitudes import mag_estimate + fails = [] obs_stats = [] agasc_stats = [] @@ -137,7 +139,8 @@ def get_agasc_id_stats_pool( obs_stats, agasc_stats, fails, failed_jobs """ import time - from astropy.table import vstack, Table + + from astropy.table import Table, vstack if obs_status_override is None: obs_status_override = {} @@ -602,7 +605,7 @@ def do( ) if len(stars_obs) == 0: - logger.info(f"There are no new observations to process") + logger.info("There are no new observations to process") return # do the processing @@ -666,7 +669,7 @@ def do( if report and len(agasc_stats): if report_date is None: report_dir = reports_dir - report_data_file = report_dir / f"report_data.pkl" + report_data_file = report_dir / "report_data.pkl" nav_links = None report_date = CxoTime.now() else: @@ -717,18 +720,18 @@ def do( }, ] - multi_star_html_args = dict( - filename="index.html", - sections=sections, - updated_stars=updated_stars, - fails=fails, - report_date=report_date.date, - tstart=start, - tstop=stop, - nav_links=nav_links, - include_all_stars=False, - no_progress=no_progress, - ) + multi_star_html_args = { + "filename": "index.html", + "sections": sections, + "updated_stars": updated_stars, + "fails": fails, + "report_date": report_date.date, + "tstart": start, + "tstop": stop, + "nav_links": nav_links, + "include_all_stars": False, + "no_progress": no_progress, + } try: report = msr.MagEstimateReport( diff --git a/agasc/supplement/utils.py b/agasc/supplement/utils.py index f5214d0e..3f14f583 100644 --- a/agasc/supplement/utils.py +++ b/agasc/supplement/utils.py @@ -1,16 +1,15 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from pathlib import Path import logging import warnings -import numpy as np +from pathlib import Path -from ska_helpers.utils import lru_cache_timed +import numpy as np import tables +from astropy.table import Table, unique, vstack from cxotime import CxoTime -from astropy.table import Table, vstack, unique - -from ..paths import SUPPLEMENT_FILENAME, default_agasc_dir +from ska_helpers.utils import lru_cache_timed +from agasc.paths import SUPPLEMENT_FILENAME, default_agasc_dir __all__ = [ "get_supplement_table", diff --git a/agasc/tests/test_agasc_2.py b/agasc/tests/test_agasc_2.py index a399741d..8d5763b6 100644 --- a/agasc/tests/test_agasc_2.py +++ b/agasc/tests/test_agasc_2.py @@ -25,10 +25,10 @@ from pathlib import Path import numpy as np +import pytest import Ska.Shell from astropy.io import ascii -from astropy.table import Table, Row -import pytest +from astropy.table import Row, Table import agasc @@ -283,7 +283,7 @@ def test_get_stars3(): def test_get_stars_many(): """Test get_stars() with at least GET_STARS_METHOD_THRESHOLD (5000) stars""" - from .. import agasc + from agasc import agasc stars = agasc.get_agasc_cone(0, 0, radius=0.5) agasc_ids = stars["AGASC_ID"] diff --git a/agasc/tests/test_agasc_healpix.py b/agasc/tests/test_agasc_healpix.py index 023faa3c..1a67173d 100644 --- a/agasc/tests/test_agasc_healpix.py +++ b/agasc/tests/test_agasc_healpix.py @@ -9,7 +9,6 @@ import agasc from agasc.healpix import get_healpix_info - AGASC_FILES = {} for root, version in [("proseco_agasc_*", "1p8"), ("agasc_healpix_*", "1p7")]: try: diff --git a/agasc/tests/test_obs_status.py b/agasc/tests/test_obs_status.py index 6119eb77..a303cff3 100644 --- a/agasc/tests/test_obs_status.py +++ b/agasc/tests/test_obs_status.py @@ -1,15 +1,16 @@ -import pytest +import builtins import io import os -import numpy as np import pathlib -from astropy import table + +import numpy as np +import pytest import tables -import builtins +from astropy import table -from agasc.supplement.magnitudes import mag_estimate, star_obs_catalogs from agasc.scripts import update_supplement -from agasc.supplement.utils import OBS_DTYPE, BAD_DTYPE +from agasc.supplement.magnitudes import mag_estimate, star_obs_catalogs +from agasc.supplement.utils import BAD_DTYPE, OBS_DTYPE TEST_DATA_DIR = pathlib.Path(__file__).parent / "data" @@ -575,7 +576,7 @@ def test_parse_file(monkeypatch, mock_open): data = update_supplement._sanitize_args(data) assert ( data == TEST_DATA[filename] - ), f"update_supplement._sanitize_args should be idempotent" + ), "update_supplement._sanitize_args should be idempotent" def test_parse_args_file(monkeypatch, mock_open): @@ -1183,7 +1184,7 @@ def test_save_version(monkeypatch): # write method is called import agasc - versions = dict(obs=agasc.__version__, mags=agasc.__version__) + versions = {"obs": agasc.__version__, "mags": agasc.__version__} def mock_write(*args, **kwargs): mock_write.calls.append((args, kwargs)) @@ -1193,7 +1194,7 @@ def mock_write(*args, **kwargs): assert len(args[0]) == 1 assert "supplement" in args[0].colnames if kwargs["path"] == "agasc_versions": - for k, v in versions.items(): + for k, _v in versions.items(): assert k in args[0].colnames assert args[0][k][0] == versions[k] diff --git a/create_derived_agasc_h5.py b/create_derived_agasc_h5.py index e721fcfe..bc48c013 100644 --- a/create_derived_agasc_h5.py +++ b/create_derived_agasc_h5.py @@ -40,8 +40,8 @@ import tables from astropy.table import Table -from agasc.healpix import get_healpix from agasc import default_agasc_dir +from agasc.healpix import get_healpix def get_parser(): diff --git a/dev/profile_get_functions.py b/dev/profile_get_functions.py index 608e360e..79df7697 100644 --- a/dev/profile_get_functions.py +++ b/dev/profile_get_functions.py @@ -1,7 +1,9 @@ -import time -import numpy as np import argparse +import time from pathlib import Path + +import numpy as np + import agasc @@ -25,7 +27,7 @@ def get_ra_decs(): def time_get_agasc_cone(agasc_file, ras, decs, n_cone, cache=False): - kwargs = dict(radius=1.0, agasc_file=agasc_file) + kwargs = {"radius": 1.0, "agasc_file": agasc_file} if cache: kwargs["cache"] = True diff --git a/dev/profile_memory.py b/dev/profile_memory.py index 25a85f39..b40dab9f 100644 --- a/dev/profile_memory.py +++ b/dev/profile_memory.py @@ -6,18 +6,18 @@ import agasc -STD_INFO = dict( - att=(0, 0, 0), - detector="ACIS-S", - sim_offset=0, - focus_offset=0, - date="2018:001", - n_guide=5, - n_fid=3, - t_ccd=-11, - man_angle=90, - dither=8.0, -) +STD_INFO = { + "att": (0, 0, 0), + "detector": "ACIS-S", + "sim_offset": 0, + "focus_offset": 0, + "date": "2018:001", + "n_guide": 5, + "n_fid": 3, + "t_ccd": -11, + "man_angle": 90, + "dither": 8.0, +} print(f"agasc.__version__ = {agasc.__version__}") diff --git a/dev/profile_memory_psrecord.py b/dev/profile_memory_psrecord.py index 82217d08..635c7d4f 100644 --- a/dev/profile_memory_psrecord.py +++ b/dev/profile_memory_psrecord.py @@ -12,20 +12,7 @@ def wait_until_time(t1): print(f"import other dependencies at {time.time() - t0:.2f} s") -import contextlib -import functools import os -import re -from packaging.version import Version -from pathlib import Path -from typing import Optional - -import numexpr -import numpy as np -import tables -from astropy.table import Column, Table -from Chandra.Time import DateTime -import tables wait_until_time(2.0) @@ -52,7 +39,6 @@ def wait_until_time(t1): wait_until_time(8.0) print(f"importing sparkles at {time.time() - t0:.2f} s") -import sparkles wait_until_time(10.0) diff --git a/scripts/make_fits.py b/scripts/make_fits.py index 2437d9c7..f8795e14 100644 --- a/scripts/make_fits.py +++ b/scripts/make_fits.py @@ -1,10 +1,10 @@ +import argparse import os from pathlib import Path + import numpy as np -import astropy.io.fits as fits +from astropy.io import fits from astropy.table import Table -import argparse - NEW_COMMENT = ( " ", diff --git a/scripts/make_links.py b/scripts/make_links.py index 61de2029..bf0c2dcf 100644 --- a/scripts/make_links.py +++ b/scripts/make_links.py @@ -1,7 +1,5 @@ import os from pathlib import Path -from Ska.Shell import bash - for agasc_dir in Path("agasc").glob("*"): updir = agasc_dir.name.upper() diff --git a/setup.py b/setup.py index bd1ddbdc..12e97d4b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from setuptools import setup import os +from setuptools import setup + try: from testr.setup_helper import cmdclass except ImportError: From 8a08f775b244507874db5cca7f74422fcb650039 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 21:16:40 -0400 Subject: [PATCH 09/10] Fix remaining ruff warnings --- agasc/scripts/supplement_tasks.py | 10 ++++++++-- agasc/supplement/magnitudes/mag_estimate.py | 12 ++++++------ agasc/supplement/magnitudes/mag_estimate_report.py | 8 ++++---- agasc/supplement/magnitudes/update_mag_supplement.py | 4 ++-- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/agasc/scripts/supplement_tasks.py b/agasc/scripts/supplement_tasks.py index 1010f11c..b4f4e227 100644 --- a/agasc/scripts/supplement_tasks.py +++ b/agasc/scripts/supplement_tasks.py @@ -50,7 +50,12 @@ def update_rc(): email_promotion_report(filenames, destdir=AGASC_DATA, to="aca@cfa.harvard.edu") subprocess.run( - ["task_schedule3.pl", "-config", "agasc/task_schedule_update_supplement_rc.cfg"] + [ + "task_schedule3.pl", + "-config", + "agasc/task_schedule_update_supplement_rc.cfg", + ], + check=False, ) @@ -65,7 +70,8 @@ def disposition(): "task_schedule3.pl", "-config", "agasc/task_schedule_supplement_dispositions.cfg", - ] + ], + check=False, ) diff --git a/agasc/supplement/magnitudes/mag_estimate.py b/agasc/supplement/magnitudes/mag_estimate.py index c32dff96..076689a1 100644 --- a/agasc/supplement/magnitudes/mag_estimate.py +++ b/agasc/supplement/magnitudes/mag_estimate.py @@ -68,10 +68,10 @@ def __init__( self.error_code = EXCEPTION_CODES[msg] self.msg = msg self.agasc_id = agasc_id - self.obsid = obsid[0] if type(obsid) is list and len(obsid) == 1 else obsid + self.obsid = obsid[0] if isinstance(obsid, list) and len(obsid) == 1 else obsid self.mp_starcat_time = ( mp_starcat_time[0] - if type(mp_starcat_time) is list and len(mp_starcat_time) == 1 + if isinstance(mp_starcat_time, list) and len(mp_starcat_time) == 1 else mp_starcat_time ) for k in kwargs: @@ -143,12 +143,12 @@ def get_responsivity(time): This was estimated with bright stars that were observed more than a hundred times during the mission. More details in the `responsivity notebook`_: - .. _responsivity notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/03-high_mag_responsivity-fit.ipynb # noqa + .. _responsivity notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/03-high_mag_responsivity-fit.ipynb :param time: float Time in CXC seconds :return: - """ + """ # noqa: E501 a, b, c = [3.19776750e-02, 5.35201479e08, 8.49670756e07] return -a * (1 + scipy.special.erf((time - b) / c)) / 2 @@ -161,12 +161,12 @@ def get_droop_systematic_shift(magnitude): The magnitude shift is time-independent. It depends only on the catalog magnitude and is zero for bright stars. More details in the `droop notebook`_: - .. _droop notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/04-DroopAfterSubtractionAndResponsivity-fit.ipynb # noqa + .. _droop notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/04-DroopAfterSubtractionAndResponsivity-fit.ipynb :param magnitude: float Catalog ACA magnitude :return: - """ + """ # noqa: E501 a, b = [11.25572, 0.59486369] return np.exp((magnitude - a) / b) diff --git a/agasc/supplement/magnitudes/mag_estimate_report.py b/agasc/supplement/magnitudes/mag_estimate_report.py index 702ee6e5..a55f2f30 100644 --- a/agasc/supplement/magnitudes/mag_estimate_report.py +++ b/agasc/supplement/magnitudes/mag_estimate_report.py @@ -409,7 +409,7 @@ def plot_agasc_id_single( ax.set_title(title) elif obsid is not None: ax.set_title(f"OBSID {obsid}") - if type(highlight_obsid) is not list and np.isscalar(highlight_obsid): + if not isinstance(highlight_obsid, list) and np.isscalar(highlight_obsid): highlight_obsid = [highlight_obsid] agasc_stat = self.agasc_stats[self.agasc_stats["agasc_id"] == agasc_id][0] @@ -456,7 +456,7 @@ def plot_agasc_id_single( timeline["std"] = np.nan timeline["mag_mean"] = np.nan timeline["mag_std"] = np.nan - for i, obsid in enumerate(np.unique(timeline["obsid"])): + for obsid in np.unique(timeline["obsid"]): sel = obs_stats["obsid"] == obsid if draw_obs_mag_stats and np.any(sel): timeline["mag_mean"][timeline["obsid"] == obsid] = obs_stats[sel][ @@ -810,7 +810,7 @@ def plot_flags(telemetry, ax=None, obsid=None): ] if obsid: - for i, (l, o) in enumerate(flags): + for i, (l, o) in enumerate(flags): # noqa: E741 flags[i] = (l, o[timeline["obsid"] == obsid]) all_ok = all_ok[timeline["obsid"] == obsid] timeline = timeline[timeline["obsid"] == obsid] @@ -823,7 +823,7 @@ def plot_flags(telemetry, ax=None, obsid=None): all_ok = timeline["obs_ok"] & all_ok flags = [("OBS not OK", ~timeline["obs_ok"])] + flags - for i, obsid in enumerate(obsids): + for obsid in obsids: limits[obsid] = ( timeline["x"][timeline["obsid"] == obsid].min(), timeline["x"][timeline["obsid"] == obsid].max(), diff --git a/agasc/supplement/magnitudes/update_mag_supplement.py b/agasc/supplement/magnitudes/update_mag_supplement.py index 481121cf..9d36346b 100755 --- a/agasc/supplement/magnitudes/update_mag_supplement.py +++ b/agasc/supplement/magnitudes/update_mag_supplement.py @@ -376,7 +376,7 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): continue mp_starcat_times = ( fail["mp_starcat_time"] - if type(fail["mp_starcat_time"]) is list + if isinstance(fail["mp_starcat_time"], list) else [fail["mp_starcat_time"]] ) agasc_id = fail["agasc_id"] @@ -457,7 +457,7 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): {%- endfor -%}] comments: {{ obs.comments }} {%- endfor %} -""" # noqa +""" # noqa: E501 tpl = jinja2.Template(yaml_template) result = tpl.render(observations=obs, agasc_ids=agasc_ids) if filename: From 0226ca8290b561d8440b6b338915e68741dc9a7e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Tue, 3 Oct 2023 21:25:54 -0400 Subject: [PATCH 10/10] Fix missing space --- agasc/agasc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agasc/agasc.py b/agasc/agasc.py index 0337a22e..131e7870 100644 --- a/agasc/agasc.py +++ b/agasc/agasc.py @@ -367,7 +367,7 @@ def sphere_dist(ra1, dec1, ra2, dec2): dec1 = np.radians(dec1).astype(np.float64) dec2 = np.radians(dec2).astype(np.float64) - numerator = numexpr.evaluate( # noqa: F841 + numerator = numexpr.evaluate( # noqa: F841 "sin((dec2 - dec1) / 2) ** 2 + " "cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2" )