diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 274d672d..10d1e568 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -24,13 +24,13 @@ Lists Lists are accepted only by the :python:`calibrator`, :python:`converter`, and :python:`generator`. These lists have to correspond to a list of source IDs. Both lists of strings and lists of long are accepted. When a list is passed to one of the tools, the function will internally request the required data for the given sources from the Gaia Archive (currently geapre), ask for -credentials (username and password), execute the function, and return the results. +Cosmos credentials (username and password), execute the function, and return the results. ADQL queries ------------ ADQL queries are accepted only by the :python:`calibrator`, :python:`converter`, and :python:`generator`. Queries need to be passed as strings (e.g.: :python:`"select TOP 100 source_id from user_dr3int6.gaia_source where has_xp_continuous = 'True'"`). -Queries are sent to the Gaia Archive (geapre), and executed after requesting credentials (username and password). +Queries are sent to the Gaia Archive (geapre), and executed after requesting Cosmos credentials (username and password). DataFrames ---------- diff --git a/gaiaxpy/calibrator/calibrator.py b/gaiaxpy/calibrator/calibrator.py index 59a85300..c6ccc4d9 100644 --- a/gaiaxpy/calibrator/calibrator.py +++ b/gaiaxpy/calibrator/calibrator.py @@ -31,7 +31,9 @@ def calibrate( output_path='.', output_file='output_spectra', output_format=None, - save_file=True): + save_file=True, + username=None, + password=None): """ Calibration utility: calibrates the input internally-calibrated continuously-represented mean spectra to the absolute system. An absolute @@ -55,6 +57,8 @@ def calibrate( input file. save_file (bool): Whether to save the output in a file. If false, output_format and output_file are ignored. + username (str): Cosmos username. + password (str): Cosmos password. Returns: (tuple): tuple containing: @@ -70,7 +74,9 @@ def calibrate( output_path, output_file, output_format, - save_file) + save_file, + username=username, + password=password) def _calibrate( @@ -82,7 +88,9 @@ def _calibrate( output_format=None, save_file=True, bp_model='v375wi', - rp_model='v142r'): + rp_model='v142r', + username=None, + password=None): """ Internal method of the calibration utility. Refer to "calibrate". @@ -99,7 +107,7 @@ def _calibrate( """ _validate_wl_sampling(sampling) _validate_arguments(_calibrate.__defaults__[3], output_file, save_file) - parsed_input_data, extension = InputReader(input_object, _calibrate)._read() + parsed_input_data, extension = InputReader(input_object, _calibrate, username, password)._read() label = 'calibrator' xp_design_matrices, xp_merge = _generate_xp_matrices_and_merge(label, sampling, bp_model, rp_model) diff --git a/gaiaxpy/converter/converter.py b/gaiaxpy/converter/converter.py index b8e8a3be..868cb7ed 100644 --- a/gaiaxpy/converter/converter.py +++ b/gaiaxpy/converter/converter.py @@ -34,7 +34,9 @@ def convert( output_path='.', output_file='output_spectra', output_format=None, - save_file=True): + save_file=True, + username=None, + password=None): """ Conversion utility: converts the input internally calibrated mean spectra from the continuous representation to a sampled form. The @@ -60,6 +62,8 @@ def convert( input file. save_file (bool): Whether to save the output in a file. If false, output_format and output_file are ignored. + username (str): Cosmos username. + password (str): Cosmos password. Returns: (tuple): tuple containing: @@ -72,7 +76,7 @@ def convert( # Check sampling _validate_pwl_sampling(sampling) _validate_arguments(convert.__defaults__[3], output_file, save_file) - parsed_input_data, extension = InputReader(input_object, convert)._read() + parsed_input_data, extension = InputReader(input_object, convert, username, password)._read() config_df = load_config(config_file) # Union of unique ids as sets unique_bases_ids = get_unique_basis_ids(parsed_input_data) diff --git a/gaiaxpy/file_parser/parse_generic.py b/gaiaxpy/file_parser/parse_generic.py index 71a07fcb..4f50e311 100644 --- a/gaiaxpy/file_parser/parse_generic.py +++ b/gaiaxpy/file_parser/parse_generic.py @@ -81,7 +81,7 @@ def parse(self, file_path): Returns: DataFrame: Pandas DataFrame representing the file. - str: File extension ('.avro', '.csv', '.fits', or '.xml'). + str: File extension ('.csv', '.fits', or '.xml'). """ extension = _get_file_extension(file_path) parser = self.get_parser(extension) diff --git a/gaiaxpy/generator/generator.py b/gaiaxpy/generator/generator.py index dc5f175a..3a0fcbe3 100644 --- a/gaiaxpy/generator/generator.py +++ b/gaiaxpy/generator/generator.py @@ -14,7 +14,9 @@ def generate( output_file='output_synthetic_photometry', output_format=None, save_file=True, - error_correction=False): + error_correction=False, + username=None, + password=None): """ Synthetic photometry utility: generates synthetic photometry in a set of available systems from the input internally-calibrated @@ -39,6 +41,8 @@ def generate( error_correction (bool): Whether to apply to the photometric errors the tabulated factors to mitigate underestimated errors (see Montegriffo et al., 2022, for more details). + username (str): Cosmos username. + password (str): Cosmos password. Returns: DataFrame: A DataFrame of all synthetic photometry results. @@ -56,7 +60,7 @@ def generate( else: raise ValueError('Parameter photometric_system must be either a PhotometricSystem or a list.') # Load data - parsed_input_data, extension = InputReader(input_object, generate)._read() + parsed_input_data, extension = InputReader(input_object, generate, username, password)._read() gaia_system = PhotometricSystem.Gaia_DR3_Vega # Create multi generator gaia_initially_in_systems = bool(gaia_system in int_photometric_system) diff --git a/gaiaxpy/input_reader/archive_reader.py b/gaiaxpy/input_reader/archive_reader.py new file mode 100644 index 00000000..c91fbf62 --- /dev/null +++ b/gaiaxpy/input_reader/archive_reader.py @@ -0,0 +1,16 @@ +from astroquery.gaia import GaiaClass + +class ArchiveReader(object): + + def __init__(self, function, user, password): + self.function = function + self.user = user + self.password = password + + def _login(self, gaia): + user = self.user + password = self.password + if user and password: + gaia.login(user=user, password=password) + else: + gaia.login() diff --git a/gaiaxpy/input_reader/dataframe_reader.py b/gaiaxpy/input_reader/dataframe_reader.py index 34d0ca25..e337ee46 100644 --- a/gaiaxpy/input_reader/dataframe_reader.py +++ b/gaiaxpy/input_reader/dataframe_reader.py @@ -1,7 +1,7 @@ from numpy import ndarray +from pandas import isnull from .dataframe_numpy_array_reader import DataFrameNumPyArrayReader from .dataframe_string_array_reader import DataFrameStringArrayReader -# TODO: move this function to core as it's now used by more than one subpackage from gaiaxpy.core import array_to_symmetric_matrix matrix_columns = [('bp_n_parameters', 'bp_coefficient_correlations'), @@ -48,6 +48,10 @@ def _read_df(self): if needs_matrix_conversion(array_columns): for index, row in data.iterrows(): for size_column, values_column in matrix_columns: - data[values_column][index] = array_to_symmetric_matrix( - data[size_column][index].astype(int), row[values_column]) + try: + data[values_column][index] = array_to_symmetric_matrix( + data[size_column][index].astype(int), row[values_column]) + except AttributeError as err: + if isnull(data[size_column][index]): + continue return data, None # No extension for dataframes diff --git a/gaiaxpy/input_reader/dataframe_string_array_reader.py b/gaiaxpy/input_reader/dataframe_string_array_reader.py index b88f1e29..10c1a11f 100644 --- a/gaiaxpy/input_reader/dataframe_string_array_reader.py +++ b/gaiaxpy/input_reader/dataframe_string_array_reader.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +from math import isnan # Avoid warning, false positive pd.options.mode.chained_assignment = None @@ -17,8 +18,13 @@ def _parse_parenthesis_arrays(self): for column in array_columns: # String column to NumPy array for index, row in df.iterrows(): - df[column][index] = np.fromstring( - row[column][1:-1], sep=',') + current_element = row[column] + try: + df[column][index] = np.fromstring( + current_element[1:-1], sep=',') + except TypeError: + if isinstance(current_element, float) and isnan(current_element): + continue return df def _parse_brackets_arrays(self): diff --git a/gaiaxpy/input_reader/input_reader.py b/gaiaxpy/input_reader/input_reader.py index a320a2eb..9cf6e6ae 100644 --- a/gaiaxpy/input_reader/input_reader.py +++ b/gaiaxpy/input_reader/input_reader.py @@ -5,18 +5,23 @@ from .list_reader import ListReader from .query_reader import QueryReader + default_extension = 'csv' class InputReader(object): - def __init__(self, content, function): + def __init__(self, content, function, user=None, password=None): self.content = content self.function = function + self.user = user + self.password = password def _string_reader(self): content = self.content function = self.function + user = self.user + password = self.password # Check whether content is path if path.isfile(content) or path.isabs(content): selector = FileReader(function) @@ -24,7 +29,7 @@ def _string_reader(self): parsed_input_data, extension = parser.parse(content) # Query should start with select elif content.lower().startswith('select'): - parsed_input_data, extension = QueryReader(content, function)._read() + parsed_input_data, extension = QueryReader(content, function, user, password)._read() else: raise ValueError('Input string does not correspond to an existing file and it is not an ADQL query.') return parsed_input_data, extension @@ -32,6 +37,8 @@ def _string_reader(self): def _read(self): content = self.content function = self.function + user = self.user + password = self.password # DataFrame reader if isinstance(content, pd.DataFrame): # Call Dataframe reader @@ -39,7 +46,7 @@ def _read(self): # List reader for query elif isinstance(content, list): # Construct query from list - parsed_data, extension = ListReader(content, function)._read() + parsed_data, extension = ListReader(content, function, user, password)._read() # String can be either query or file path elif isinstance(content, str): parsed_data, extension = self._string_reader() diff --git a/gaiaxpy/input_reader/list_reader.py b/gaiaxpy/input_reader/list_reader.py index 95b3bbb3..eecaaedb 100644 --- a/gaiaxpy/input_reader/list_reader.py +++ b/gaiaxpy/input_reader/list_reader.py @@ -1,5 +1,6 @@ from astroquery.gaia import GaiaClass from .dataframe_reader import DataFrameReader +from .archive_reader import ArchiveReader not_supported_functions = ['apply_colour_equation'] @@ -13,14 +14,14 @@ def extremes_are_enclosing(first_row, column): return False -class ListReader(object): +class ListReader(ArchiveReader): - def __init__(self, content, function): + def __init__(self, content, function, user, password): + super(ListReader, self).__init__(function, user, password) if content != []: self.content = content else: raise ValueError('Input list cannot be empty.') - self.function = function def _read(self): sources = self.content @@ -29,7 +30,7 @@ def _read(self): raise ValueError(f'Function {function_name} does not support receiving a list as input.') # Connect to geapre gaia = GaiaClass(gaia_tap_server='https://geapre.esac.esa.int/', gaia_data_server='https://geapre.esac.esa.int/') - gaia.login() + self._login(gaia) # ADQL query result = gaia.load_data(ids=sources, format='csv', data_release='Gaia DR3_INT6', data_structure='raw', retrieval_type='XP_CONTINUOUS', avoid_datatype_check=True) diff --git a/gaiaxpy/input_reader/query_reader.py b/gaiaxpy/input_reader/query_reader.py index 6ee9c0b9..2b509caa 100644 --- a/gaiaxpy/input_reader/query_reader.py +++ b/gaiaxpy/input_reader/query_reader.py @@ -1,14 +1,15 @@ from astroquery.gaia import GaiaClass from .dataframe_reader import DataFrameReader +from .archive_reader import ArchiveReader not_supported_functions = ['apply_colour_equation'] -class QueryReader(object): +class QueryReader(ArchiveReader): - def __init__(self, content, function): + def __init__(self, content, function, user=None, password=None): self.content = content - self.function = function + super(QueryReader, self).__init__(function, user, password) def _read(self, data_release='Gaia DR3_INT6'): query = self.content @@ -17,7 +18,7 @@ def _read(self, data_release='Gaia DR3_INT6'): raise ValueError(f'Function {function_name} does not support receiving a query as input.') # Connect to geapre gaia = GaiaClass(gaia_tap_server='https://geapre.esac.esa.int/', gaia_data_server='https://geapre.esac.esa.int/') - gaia.login() + self._login(gaia) # ADQL query job = gaia.launch_job_async(query, dump_to_file=False) ids = job.get_results()