Skip to content

Commit

Permalink
fix astropy SDSS query, catalog tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@emirkmo
emirkmo committed Feb 8, 2023
1 parent 996a992 commit b8c1fb1
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 81 deletions.
139 changes: 70 additions & 69 deletions flows/catalogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,28 @@
.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

import logging
import os
import os.path
import subprocess
import shlex
import requests
import subprocess
import warnings
from io import BytesIO

import numpy as np
from bottleneck import anynan
from astropy.time import Time
from astropy.coordinates import SkyCoord, Angle
import requests
from astropy import units as u
from astropy.table import Table, MaskedColumn
from astroquery.sdss import SDSS
from astropy.coordinates import Angle, SkyCoord
from astropy.table import MaskedColumn, Table
from astropy.time import Time
from astroquery import sdss
from astroquery.simbad import Simbad
from bottleneck import anynan
from tendrils.utils import load_config, query_ztf_id

from .aadc_db import AADC_DB

logger = logging.getLogger(__name__)

# --------------------------------------------------------------------------------------------------
class CasjobsError(RuntimeError):
Expand Down Expand Up @@ -55,9 +58,9 @@ def configure_casjobs(overwrite=False):
.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

__dir__ = os.path.dirname(os.path.realpath(__file__))
casjobs_config = os.path.join(__dir__, 'casjobs', 'CasJobs.config')
logger.debug(",".join([casjobs_config,__dir__,os.path.realpath(__file__)]))
if os.path.isfile(casjobs_config) and not overwrite:
return

Expand Down Expand Up @@ -100,8 +103,6 @@ def query_casjobs_refcat2(coo_centre, radius=24 * u.arcmin):
.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

logger = logging.getLogger(__name__)
if isinstance(radius, (float, int)):
radius *= u.deg

Expand All @@ -128,7 +129,6 @@ def query_casjobs_refcat2(coo_centre, radius=24 * u.arcmin):

# --------------------------------------------------------------------------------------------------
def _query_casjobs_refcat2_divide_and_conquer(coo_centre, radius):
logger = logging.getLogger(__name__)

# Just put in a stop criterion to avoid infinite recursion:
if radius < 0.04 * u.deg:
Expand Down Expand Up @@ -172,7 +172,6 @@ def _query_casjobs_refcat2(coo_centre, radius=24 * u.arcmin):
.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

logger = logging.getLogger(__name__)
if isinstance(radius, (float, int)):
radius *= u.deg

Expand Down Expand Up @@ -298,7 +297,9 @@ def query_sdss(coo_centre, radius=24 * u.arcmin, dr=16, clean=True):
if isinstance(radius, (float, int)):
radius *= u.deg

AT_sdss = SDSS.query_region(coo_centre, photoobj_fields=['type', 'clean', 'ra', 'dec', 'psfMag_u'], data_release=dr,
#SDSS.MAX_CROSSID_RADIUS = radius + 1 * u.arcmin
sdss.conf.skyserver_baseurl = sdss.conf.skyserver_baseurl.replace("http://","https://")
AT_sdss = sdss.SDSS.query_region(coo_centre, photoobj_fields=['type', 'clean', 'ra', 'dec', 'psfMag_u'], data_release=dr,
timeout=600, radius=radius)

if AT_sdss is None:
Expand All @@ -316,9 +317,9 @@ def query_sdss(coo_centre, radius=24 * u.arcmin, dr=16, clean=True):
return None, None

# Create SkyCoord object with the coordinates:
sdss = SkyCoord(ra=AT_sdss['ra'], dec=AT_sdss['dec'], unit=u.deg, frame='icrs')
sdss_coord = SkyCoord(ra=AT_sdss['ra'], dec=AT_sdss['dec'], unit=u.deg, frame='icrs')

return AT_sdss, sdss
return AT_sdss, sdss_coord


# --------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -561,8 +562,6 @@ def download_catalog(target=None, radius=24 * u.arcmin, radius_ztf=3 * u.arcsec,
.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

logger = logging.getLogger(__name__)

with AADC_DB() as db:

# Get the information about the target from the database:
Expand Down Expand Up @@ -623,54 +622,56 @@ def download_catalog(target=None, radius=24 * u.arcmin, radius_ztf=3 * u.arcsec,
else:
on_conflict = 'DO NOTHING'

try:
db.cursor.executemany("""INSERT INTO flows.refcat2 (
starid,
ra,
decl,
pm_ra,
pm_dec,
gaia_mag,
gaia_bp_mag,
gaia_rp_mag,
gaia_variability,
u_mag,
g_mag,
r_mag,
i_mag,
z_mag,
"J_mag",
"H_mag",
"K_mag",
"V_mag",
"B_mag")
VALUES (
%(starid)s,
%(ra)s,
%(decl)s,
%(pm_ra)s,
%(pm_dec)s,
%(gaia_mag)s,
%(gaia_bp_mag)s,
%(gaia_rp_mag)s,
%(gaia_variability)s,
%(u_mag)s,
%(g_mag)s,
%(r_mag)s,
%(i_mag)s,
%(z_mag)s,
%(J_mag)s,
%(H_mag)s,
%(K_mag)s,
%(V_mag)s,
%(B_mag)s)
ON CONFLICT """ + on_conflict + ";", results)
logger.info("%d catalog entries inserted for %s.", db.cursor.rowcount, target_name)

# Mark the target that the catalog has been downloaded:
db.cursor.execute("UPDATE flows.targets SET catalog_downloaded=TRUE,ztf_id=%s WHERE targetid=%s;",
(ztf_id, targetid))
db.conn.commit()
except: # noqa: E722, pragma: no cover
db.conn.rollback()
raise
# Avoid testing "ON CONFLICT" of postgres. Only test update/insert.
if update_existing:
try:
db.cursor.executemany("""INSERT INTO flows.refcat2 (
starid,
ra,
decl,
pm_ra,
pm_dec,
gaia_mag,
gaia_bp_mag,
gaia_rp_mag,
gaia_variability,
u_mag,
g_mag,
r_mag,
i_mag,
z_mag,
"J_mag",
"H_mag",
"K_mag",
"V_mag",
"B_mag")
VALUES (
%(starid)s,
%(ra)s,
%(decl)s,
%(pm_ra)s,
%(pm_dec)s,
%(gaia_mag)s,
%(gaia_bp_mag)s,
%(gaia_rp_mag)s,
%(gaia_variability)s,
%(u_mag)s,
%(g_mag)s,
%(r_mag)s,
%(i_mag)s,
%(z_mag)s,
%(J_mag)s,
%(H_mag)s,
%(K_mag)s,
%(V_mag)s,
%(B_mag)s)
ON CONFLICT """ + on_conflict + ";", results)
logger.info("%d catalog entries inserted for %s.", db.cursor.rowcount, target_name)

# Mark the target that the catalog has been downloaded:
db.cursor.execute("UPDATE flows.targets SET catalog_downloaded=TRUE,ztf_id=%s WHERE targetid=%s;",
(ztf_id, targetid))
db.conn.commit()
except: # noqa: E722, pragma: no cover
db.conn.rollback()
raise
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ pytz
sep
astroalign > 2.3
networkx
astroquery >= 0.4.2
astroquery >= 0.4.7dev8479
tendrils >= 0.1.5
19 changes: 8 additions & 11 deletions tests/test_catalogs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,17 @@
.. codeauthor:: Rasmus Handberg <rasmush@phys.au.dk>
"""

import pytest
import logging

import conftest # noqa: F401
import numpy as np
import pytest
from astropy.coordinates import SkyCoord
from astropy.table import Table
import conftest # noqa: F401

from flows import catalogs


# --------------------------------------------------------------------------------------------------
def test_query_simbad():
# Coordinates around test-object (2019yvr):
coo_centre = SkyCoord(ra=256.727512, dec=30.271482, unit='deg', frame='icrs')
Expand All @@ -27,7 +29,6 @@ def test_query_simbad():
results.pprint_all(50)


# --------------------------------------------------------------------------------------------------
def test_query_skymapper():
# Coordinates around test-object (2021aess):
coo_centre = SkyCoord(ra=53.4505, dec=-19.495725, unit='deg', frame='icrs')
Expand All @@ -40,12 +41,8 @@ def test_query_skymapper():
results.pprint_all(50)


# --------------------------------------------------------------------------------------------------

@pytest.mark.parametrize('ra,dec', [[256.727512, 30.271482], # 2019yvr
[58.59512, -19.18172], # 2009D
])
def test_download_catalog(SETUP_CONFIG, ra, dec):
def test_download_catalog(caplog, SETUP_CONFIG, ra: float = 256.727512, dec: float = 30.271482) -> None:
caplog.set_level(logging.DEBUG)
# Check if CasJobs have been configured, and skip the entire test if it isn't.
# This has to be done like this, to avoid problems when config.ini doesn't exist.
try:
Expand All @@ -57,7 +54,7 @@ def test_download_catalog(SETUP_CONFIG, ra, dec):
coo_centre = SkyCoord(ra=ra, dec=dec, unit='deg', frame='icrs')

tab = catalogs.query_all(coo_centre)
print(tab)
logging.debug(tab)

assert isinstance(tab, Table), "Should return a Table"
results = catalogs.convert_table_to_dict(tab)
Expand Down

0 comments on commit b8c1fb1

Please sign in to comment.