From 27a53dae90c96e519650bcb8f54d59e8591b8102 Mon Sep 17 00:00:00 2001 From: Andy Gee Date: Mon, 12 Jul 2021 16:53:10 -0700 Subject: [PATCH] add files for psql ssl support --- bin/config_helper.py | 377 ++++++++++++++++++ bin/creds.json | 7 + bin/indexd_settings.py | 67 ++++ deployment/Secrets/config_helper.py | 372 +++++++++++++++++ deployment/Secrets/indexd_creds.json | 7 + deployment/Secrets/indexd_settings.py | 68 ++++ deployment/scripts/indexd/indexd_setup.sh | 14 + .../scripts/postgresql/postgresql_init.sql | 34 ++ deployment/uwsgi/uwsgi.ini | 4 + tests/default_test_settings.py | 73 ++-- 10 files changed, 990 insertions(+), 33 deletions(-) create mode 100644 bin/config_helper.py create mode 100644 bin/creds.json create mode 100644 bin/indexd_settings.py create mode 100644 deployment/Secrets/config_helper.py create mode 100644 deployment/Secrets/indexd_creds.json create mode 100644 deployment/Secrets/indexd_settings.py create mode 100644 deployment/scripts/indexd/indexd_setup.sh create mode 100644 deployment/scripts/postgresql/postgresql_init.sql diff --git a/bin/config_helper.py b/bin/config_helper.py new file mode 100644 index 000000000..06a2cb52d --- /dev/null +++ b/bin/config_helper.py @@ -0,0 +1,377 @@ +import json +import os +import copy +import argparse +import re +import types + +# +# make it easy to change this for testing +XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") + + +def default_search_folders(app_name): + """ + Return the list of folders to search for configuration files + """ + return [ + "%s/cdis/%s" % (XDG_DATA_HOME, app_name), + "/usr/share/cdis/%s" % app_name, + "%s/gen3/%s" % (XDG_DATA_HOME, app_name), + "/usr/share/gen3/%s" % app_name, + "/var/www/%s" % app_name, + "/etc/gen3/%s" % app_name, + os.getcwd(), + os.path.dirname(__file__), + ] + + +def find_paths(file_name, app_name, search_folders=None): + """ + Search the given folders for file_name + search_folders defaults to default_search_folders if not specified + return the first path to file_name found + """ + search_folders = search_folders or default_search_folders(app_name) + possible_files = [os.path.join(folder, file_name) for folder in search_folders] + return [path for path in possible_files if os.path.exists(path)] + + +def load_json(file_name, app_name, search_folders=None): + """ + json.load(file_name) after finding file_name in search_folders + return the loaded json data or None if file not found + """ + actual_files = find_paths(file_name, app_name, search_folders) + if not actual_files: + return None + with open(actual_files[0], "r") as reader: + return json.load(reader) + + +def inject_creds_into_fence_config(creds_file_path, config_file_path): + creds_file = open(creds_file_path, "r") + creds = json.load(creds_file) + creds_file.close() + + # get secret values from creds.json file + db_host = _get_nested_value(creds, "db_host") + db_username = _get_nested_value(creds, "db_username") + db_password = _get_nested_value(creds, "db_password") + db_database = _get_nested_value(creds, "db_database") + hostname = _get_nested_value(creds, "hostname") + indexd_password = _get_nested_value(creds, "indexd_password") + google_client_secret = _get_nested_value(creds, "google_client_secret") + google_client_id = _get_nested_value(creds, "google_client_id") + hmac_key = _get_nested_value(creds, "hmac_key") + db_path = "postgresql://{}:{}@{}:5432/{}".format( + db_username, db_password, db_host, db_database + ) + + config_file = open(config_file_path, "r").read() + + print(" DB injected with value(s) from creds.json") + config_file = _replace(config_file, "DB", db_path) + + print(" BASE_URL injected with value(s) from creds.json") + config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) + + print(" INDEXD_PASSWORD injected with value(s) from creds.json") + config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) + config_file = _replace(config_file, "INDEXD_USERNAME", "fence") + + print(" ENCRYPTION_KEY injected with value(s) from creds.json") + config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) + + print( + " OPENID_CONNECT/google/client_secret injected with value(s) " + "from creds.json" + ) + config_file = _replace( + config_file, + "OPENID_CONNECT/google/client_secret", + google_client_secret, + ) + + print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") + config_file = _replace( + config_file, "OPENID_CONNECT/google/client_id", google_client_id + ) + + open(config_file_path, "w+").write(config_file) + + +def set_prod_defaults(config_file_path): + config_file = open(config_file_path, "r").read() + + print( + " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " + "var/www/fence/fence_google_app_creds_secret.json" + ) + config_file = _replace( + config_file, + "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", + "/var/www/fence/fence_google_app_creds_secret.json", + ) + + print( + " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " + "var/www/fence/fence_google_storage_creds_secret.json" + ) + config_file = _replace( + config_file, + "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", + "/var/www/fence/fence_google_storage_creds_secret.json", + ) + + print(" INDEXD set as http://indexd-service/") + config_file = _replace(config_file, "INDEXD", "http://indexd-service/") + + print(" ARBORIST set as http://arborist-service/") + config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") + + print(" HTTP_PROXY/host set as cloud-proxy.internal.io") + config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") + + print(" HTTP_PROXY/port set as 3128") + config_file = _replace(config_file, "HTTP_PROXY/port", 3128) + + print(" DEBUG set to false") + config_file = _replace(config_file, "DEBUG", False) + + print(" MOCK_AUTH set to false") + config_file = _replace(config_file, "MOCK_AUTH", False) + + print(" MOCK_GOOGLE_AUTH set to false") + config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) + + print(" AUTHLIB_INSECURE_TRANSPORT set to true") + config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) + + print(" SESSION_COOKIE_SECURE set to true") + config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) + + print(" ENABLE_CSRF_PROTECTION set to true") + config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) + + open(config_file_path, "w+").write(config_file) + + +def inject_other_files_into_fence_config(other_files, config_file_path): + additional_cfgs = _get_all_additional_configs(other_files) + + config_file = open(config_file_path, "r").read() + + for key, value in additional_cfgs.iteritems(): + print(" {} set to {}".format(key, value)) + config_file = _nested_replace(config_file, key, value) + + open(config_file_path, "w+").write(config_file) + + +def _get_all_additional_configs(other_files): + """ + Attempt to parse given list of files + and extract configuration variables and values + """ + additional_configs = dict() + for file_path in other_files: + try: + file_ext = file_path.strip().split(".")[-1] + if file_ext == "json": + json_file = open(file_path, "r") + configs = json.load(json_file) + json_file.close() + elif file_ext == "py": + configs = from_pyfile(file_path) + else: + print( + "Cannot load config vars from a file with extention: {}".format( + file_ext + ) + ) + except Exception as exc: + # if there's any issue reading the file, exit + print( + "Error reading {}. Cannot get configuration. Skipping this file. " + "Details: {}".format(other_files, str(exc)) + ) + continue + + if configs: + additional_configs.update(configs) + + return additional_configs + + +def _nested_replace(config_file, key, value, replacement_path=None): + replacement_path = replacement_path or key + try: + for inner_key, inner_value in value.iteritems(): + temp_path = replacement_path + temp_path = temp_path + "/" + inner_key + config_file = _nested_replace( + config_file, inner_key, inner_value, temp_path + ) + except AttributeError: + # not a dict so replace + if value is not None: + config_file = _replace(config_file, replacement_path, value) + + return config_file + + +def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): + """ + Replace a nested value in a YAML file string with the given value without + losing comments. Uses a regex to do the replacement. + Args: + yaml_config (str): a string representing a full configuration file + path_to_key (str): nested/path/to/key. The value of this key will be + replaced + replacement_value (str): Replacement value for the key from + path_to_key + """ + nested_path_to_replace = path_to_key.split("/") + + # our regex looks for a specific number of spaces to ensure correct + # level of nesting. It matches to the end of the line + search_string = ( + " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" + ) + matches = re.search(search_string, yaml_config[start:]) + + # early return if we haven't found anything + if not matches: + return yaml_config + + # if we're on the last item in the path, we need to get the value and + # replace it in the original file + if len(nested_path_to_replace) == 1: + # replace the current key:value with the new replacement value + match_start = start + matches.start(0) + len(" " * nested_level) + match_end = start + matches.end(0) + yaml_config = ( + yaml_config[:match_start] + + "{}: {}\n".format( + nested_path_to_replace[0], + _get_yaml_replacement_value(replacement_value, nested_level), + ) + + yaml_config[match_end:] + ) + + return yaml_config + + # set new start point to past current match and move on to next match + start = matches.end(0) + nested_level += 1 + del nested_path_to_replace[0] + + return _replace( + yaml_config, + "/".join(nested_path_to_replace), + replacement_value, + start, + nested_level, + ) + + +def from_pyfile(filename, silent=False): + """ + Modeled after flask's ability to load in python files: + https://github.com/pallets/flask/blob/master/flask/config.py + Some alterations were made but logic is essentially the same + """ + filename = os.path.abspath(filename) + d = types.ModuleType("config") + d.__file__ = filename + try: + with open(filename, mode="rb") as config_file: + exec( # nosec pylint: disable=W0122 + compile(config_file.read(), filename, "exec"), d.__dict__ + ) + except IOError as e: + print("Unable to load configuration file ({})".format(e.strerror)) + if silent: + return False + raise + return _from_object(d) + + +def _from_object(obj): + configs = {} + for key in dir(obj): + if key.isupper(): + configs[key] = getattr(obj, key) + return configs + + +def _get_yaml_replacement_value(value, nested_level=0): + if isinstance(value, str): + return "'" + value + "'" + elif isinstance(value, bool): + return str(value).lower() + elif isinstance(value, list) or isinstance(value, set): + output = "" + for item in value: + # spaces for nested level then spaces and hyphen for each list item + output += ( + "\n" + + " " * nested_level + + " - " + + _get_yaml_replacement_value(item) + + "" + ) + return output + else: + return value + + +def _get_nested_value(dictionary, nested_path): + """ + Return a value from a dictionary given a path-like nesting of keys. + Will default to an empty string if value cannot be found. + Args: + dictionary (dict): a dictionary + nested_path (str): nested/path/to/key + Returns: + ?: Value from dict + """ + replacement_value_path = nested_path.split("/") + replacement_value = copy.deepcopy(dictionary) + + for item in replacement_value_path: + replacement_value = replacement_value.get(item, {}) + + if replacement_value == {}: + replacement_value = "" + + return replacement_value + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-i", + "--creds_file_to_inject", + default="creds.json", + help="creds file to inject into the configuration yaml", + ) + parser.add_argument( + "--other_files_to_inject", + nargs="+", + help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " + "inject into the configuration yaml", + ) + parser.add_argument( + "-c", "--config_file", default="config.yaml", help="configuration yaml" + ) + args = parser.parse_args() + + inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) + set_prod_defaults(args.config_file) + + if args.other_files_to_inject: + inject_other_files_into_fence_config( + args.other_files_to_inject, args.config_file + ) diff --git a/bin/creds.json b/bin/creds.json new file mode 100644 index 000000000..f88efd43c --- /dev/null +++ b/bin/creds.json @@ -0,0 +1,7 @@ +{ + "db_host": "localhost", + "db_username": "indexduser", + "db_password": "indexdpassword", + "db_database": "indexd_db", + "fence_database": "fence_db" + } \ No newline at end of file diff --git a/bin/indexd_settings.py b/bin/indexd_settings.py new file mode 100644 index 000000000..364aac2b9 --- /dev/null +++ b/bin/indexd_settings.py @@ -0,0 +1,67 @@ +from os import environ +import json +import config_helper +from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver +from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver +from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver + + +APP_NAME = "indexd" + + +def load_json(file_name): + return config_helper.load_json(file_name, APP_NAME) + + +conf_data = load_json("creds.json") +usr = conf_data.get("db_username", "{{db_username}}") +db = conf_data.get("db_database", "{{db_database}}") +psw = conf_data.get("db_password", "{{db_password}}") +pghost = conf_data.get("db_host", "{{db_host}}") +pgport = 5432 +index_config = conf_data.get("index_config") +CONFIG = {} + +CONFIG["JSONIFY_PRETTYPRINT_REGULAR"] = False + +dist = environ.get("DIST", None) +if dist: + CONFIG["DIST"] = json.loads(dist) + +CONFIG["INDEX"] = { + "driver": SQLAlchemyIndexDriver( + "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( + usr=usr, + psw=psw, + pghost=pghost, + pgport=pgport, + db=db, + ), + index_config=index_config, + ), +} + +CONFIG["ALIAS"] = { + "driver": SQLAlchemyAliasDriver( + "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( + usr=usr, + psw=psw, + pghost=pghost, + pgport=pgport, + db=db, + ) + ), +} + +AUTH = SQLAlchemyAuthDriver( + "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( + usr=usr, + psw=psw, + pghost=pghost, + pgport=pgport, + db=db, + ), + arborist="http://localhost/", +) + +settings = {"config": CONFIG, "auth": AUTH} diff --git a/deployment/Secrets/config_helper.py b/deployment/Secrets/config_helper.py new file mode 100644 index 000000000..22f7496e0 --- /dev/null +++ b/deployment/Secrets/config_helper.py @@ -0,0 +1,372 @@ +import json +import os +import copy +import argparse +import re +import types + +# +# make it easy to change this for testing +XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", "/usr/share/") + + +def default_search_folders(app_name): + """ + Return the list of folders to search for configuration files + """ + return [ + "%s/cdis/%s" % (XDG_DATA_HOME, app_name), + "/usr/share/cdis/%s" % app_name, + "%s/gen3/%s" % (XDG_DATA_HOME, app_name), + "/usr/share/gen3/%s" % app_name, + "/var/www/%s" % app_name, + "/etc/gen3/%s" % app_name, + ] + + +def find_paths(file_name, app_name, search_folders=None): + """ + Search the given folders for file_name + search_folders defaults to default_search_folders if not specified + return the first path to file_name found + """ + search_folders = search_folders or default_search_folders(app_name) + possible_files = [os.path.join(folder, file_name) for folder in search_folders] + return [path for path in possible_files if os.path.exists(path)] + + +def load_json(file_name, app_name, search_folders=None): + """ + json.load(file_name) after finding file_name in search_folders + return the loaded json data or None if file not found + """ + actual_files = find_paths(file_name, app_name, search_folders) + if not actual_files: + return None + with open(actual_files[0], "r") as reader: + return json.load(reader) + + +def inject_creds_into_fence_config(creds_file_path, config_file_path): + creds_file = open(creds_file_path, "r") + creds = json.load(creds_file) + creds_file.close() + + # get secret values from creds.json file + db_host = _get_nested_value(creds, "db_host") + db_username = _get_nested_value(creds, "db_username") + db_password = _get_nested_value(creds, "db_password") + db_database = _get_nested_value(creds, "db_database") + hostname = _get_nested_value(creds, "hostname") + indexd_password = _get_nested_value(creds, "indexd_password") + google_client_secret = _get_nested_value(creds, "google_client_secret") + google_client_id = _get_nested_value(creds, "google_client_id") + hmac_key = _get_nested_value(creds, "hmac_key") + db_path = "postgresql://{}:{}@{}:5432/{}".format( + db_username, db_password, db_host, db_database + ) + + config_file = open(config_file_path, "r").read() + + print(" DB injected with value(s) from creds.json") + config_file = _replace(config_file, "DB", db_path) + + print(" BASE_URL injected with value(s) from creds.json") + config_file = _replace(config_file, "BASE_URL", "https://{}/user".format(hostname)) + + print(" INDEXD_PASSWORD injected with value(s) from creds.json") + config_file = _replace(config_file, "INDEXD_PASSWORD", indexd_password) + config_file = _replace(config_file, "INDEXD_USERNAME", "fence") + + print(" ENCRYPTION_KEY injected with value(s) from creds.json") + config_file = _replace(config_file, "ENCRYPTION_KEY", hmac_key) + + print( + " OPENID_CONNECT/google/client_secret injected with value(s) " + "from creds.json" + ) + config_file = _replace( + config_file, "OPENID_CONNECT/google/client_secret", google_client_secret + ) + + print(" OPENID_CONNECT/google/client_id injected with value(s) from creds.json") + config_file = _replace( + config_file, "OPENID_CONNECT/google/client_id", google_client_id + ) + + open(config_file_path, "w+").write(config_file) + + +def set_prod_defaults(config_file_path): + config_file = open(config_file_path, "r").read() + + print( + " CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS set as " + "var/www/fence/fence_google_app_creds_secret.json" + ) + config_file = _replace( + config_file, + "CIRRUS_CFG/GOOGLE_APPLICATION_CREDENTIALS", + "/var/www/fence/fence_google_app_creds_secret.json", + ) + + print( + " CIRRUS_CFG/GOOGLE_STORAGE_CREDS set as " + "var/www/fence/fence_google_storage_creds_secret.json" + ) + config_file = _replace( + config_file, + "CIRRUS_CFG/GOOGLE_STORAGE_CREDS", + "/var/www/fence/fence_google_storage_creds_secret.json", + ) + + print(" INDEXD set as http://indexd-service/") + config_file = _replace(config_file, "INDEXD", "http://indexd-service/") + + print(" ARBORIST set as http://arborist-service/") + config_file = _replace(config_file, "ARBORIST", "http://arborist-service/") + + print(" HTTP_PROXY/host set as cloud-proxy.internal.io") + config_file = _replace(config_file, "HTTP_PROXY/host", "cloud-proxy.internal.io") + + print(" HTTP_PROXY/port set as 3128") + config_file = _replace(config_file, "HTTP_PROXY/port", 3128) + + print(" DEBUG set to false") + config_file = _replace(config_file, "DEBUG", False) + + print(" MOCK_AUTH set to false") + config_file = _replace(config_file, "MOCK_AUTH", False) + + print(" MOCK_GOOGLE_AUTH set to false") + config_file = _replace(config_file, "MOCK_GOOGLE_AUTH", False) + + print(" AUTHLIB_INSECURE_TRANSPORT set to true") + config_file = _replace(config_file, "AUTHLIB_INSECURE_TRANSPORT", True) + + print(" SESSION_COOKIE_SECURE set to true") + config_file = _replace(config_file, "SESSION_COOKIE_SECURE", True) + + print(" ENABLE_CSRF_PROTECTION set to true") + config_file = _replace(config_file, "ENABLE_CSRF_PROTECTION", True) + + open(config_file_path, "w+").write(config_file) + + +def inject_other_files_into_fence_config(other_files, config_file_path): + additional_cfgs = _get_all_additional_configs(other_files) + + config_file = open(config_file_path, "r").read() + + for key, value in additional_cfgs.iteritems(): + print(" {} set to {}".format(key, value)) + config_file = _nested_replace(config_file, key, value) + + open(config_file_path, "w+").write(config_file) + + +def _get_all_additional_configs(other_files): + """ + Attempt to parse given list of files and extract configuration variables and values + """ + additional_configs = dict() + for file_path in other_files: + try: + file_ext = file_path.strip().split(".")[-1] + if file_ext == "json": + json_file = open(file_path, "r") + configs = json.load(json_file) + json_file.close() + elif file_ext == "py": + configs = from_pyfile(file_path) + else: + print( + "Cannot load config vars from a file with extention: {}".format( + file_ext + ) + ) + except Exception as exc: + # if there's any issue reading the file, exit + print( + "Error reading {}. Cannot get configuration. Skipping this file. " + "Details: {}".format(other_files, str(exc)) + ) + continue + + if configs: + additional_configs.update(configs) + + return additional_configs + + +def _nested_replace(config_file, key, value, replacement_path=None): + replacement_path = replacement_path or key + try: + for inner_key, inner_value in value.iteritems(): + temp_path = replacement_path + temp_path = temp_path + "/" + inner_key + config_file = _nested_replace( + config_file, inner_key, inner_value, temp_path + ) + except AttributeError: + # not a dict so replace + if value is not None: + config_file = _replace(config_file, replacement_path, value) + + return config_file + + +def _replace(yaml_config, path_to_key, replacement_value, start=0, nested_level=0): + """ + Replace a nested value in a YAML file string with the given value without + losing comments. Uses a regex to do the replacement. + Args: + yaml_config (str): a string representing a full configuration file + path_to_key (str): nested/path/to/key. The value of this key will be + replaced + replacement_value (str): Replacement value for the key from + path_to_key + """ + nested_path_to_replace = path_to_key.split("/") + + # our regex looks for a specific number of spaces to ensure correct + # level of nesting. It matches to the end of the line + search_string = ( + " " * nested_level + ".*" + nested_path_to_replace[0] + "(')?(\")?:.*\n" + ) + matches = re.search(search_string, yaml_config[start:]) + + # early return if we haven't found anything + if not matches: + return yaml_config + + # if we're on the last item in the path, we need to get the value and + # replace it in the original file + if len(nested_path_to_replace) == 1: + # replace the current key:value with the new replacement value + match_start = start + matches.start(0) + len(" " * nested_level) + match_end = start + matches.end(0) + yaml_config = ( + yaml_config[:match_start] + + "{}: {}\n".format( + nested_path_to_replace[0], + _get_yaml_replacement_value(replacement_value, nested_level), + ) + + yaml_config[match_end:] + ) + + return yaml_config + + # set new start point to past current match and move on to next match + start = matches.end(0) + nested_level += 1 + del nested_path_to_replace[0] + + return _replace( + yaml_config, + "/".join(nested_path_to_replace), + replacement_value, + start, + nested_level, + ) + + +def from_pyfile(filename, silent=False): + """ + Modeled after flask's ability to load in python files: + https://github.com/pallets/flask/blob/master/flask/config.py + Some alterations were made but logic is essentially the same + """ + filename = os.path.abspath(filename) + d = types.ModuleType("config") + d.__file__ = filename + try: + with open(filename, mode="rb") as config_file: + exec( # nosec pylint: disable=W0122 + compile(config_file.read(), filename, "exec"), d.__dict__ + ) + except IOError as e: + print("Unable to load configuration file ({})".format(e.strerror)) + if silent: + return False + raise + return _from_object(d) + + +def _from_object(obj): + configs = {} + for key in dir(obj): + if key.isupper(): + configs[key] = getattr(obj, key) + return configs + + +def _get_yaml_replacement_value(value, nested_level=0): + if isinstance(value, str): + return "'" + value + "'" + elif isinstance(value, bool): + return str(value).lower() + elif isinstance(value, list) or isinstance(value, set): + output = "" + for item in value: + # spaces for nested level then spaces and hyphen for each list item + output += ( + "\n" + + " " * nested_level + + " - " + + _get_yaml_replacement_value(item) + + "" + ) + return output + else: + return value + + +def _get_nested_value(dictionary, nested_path): + """ + Return a value from a dictionary given a path-like nesting of keys. + Will default to an empty string if value cannot be found. + Args: + dictionary (dict): a dictionary + nested_path (str): nested/path/to/key + Returns: + ?: Value from dict + """ + replacement_value_path = nested_path.split("/") + replacement_value = copy.deepcopy(dictionary) + + for item in replacement_value_path: + replacement_value = replacement_value.get(item, {}) + + if replacement_value == {}: + replacement_value = "" + + return replacement_value + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-i", + "--creds_file_to_inject", + default="creds.json", + help="creds file to inject into the configuration yaml", + ) + parser.add_argument( + "--other_files_to_inject", + nargs="+", + help="fence_credentials.json, local_settings.py, fence_settings.py file(s) to " + "inject into the configuration yaml", + ) + parser.add_argument( + "-c", "--config_file", default="config.yaml", help="configuration yaml" + ) + args = parser.parse_args() + + inject_creds_into_fence_config(args.creds_file_to_inject, args.config_file) + set_prod_defaults(args.config_file) + + if args.other_files_to_inject: + inject_other_files_into_fence_config( + args.other_files_to_inject, args.config_file + ) diff --git a/deployment/Secrets/indexd_creds.json b/deployment/Secrets/indexd_creds.json new file mode 100644 index 000000000..a251cb167 --- /dev/null +++ b/deployment/Secrets/indexd_creds.json @@ -0,0 +1,7 @@ +{ + "db_host": "postgres", + "db_username": "indexd_user", + "db_password": "indexd_pass", + "db_database": "indexd_db", + "fence_database": "fence_db" + } \ No newline at end of file diff --git a/deployment/Secrets/indexd_settings.py b/deployment/Secrets/indexd_settings.py new file mode 100644 index 000000000..4b35c567a --- /dev/null +++ b/deployment/Secrets/indexd_settings.py @@ -0,0 +1,68 @@ +from os import environ +import json +import config_helper +from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver +from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver +from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver + + +APP_NAME = "indexd" + + +def load_json(file_name): + return config_helper.load_json(file_name, APP_NAME) + + +conf_data = load_json("creds.json") + +usr = conf_data.get("db_username", "{{db_username}}") +db = conf_data.get("db_database", "{{db_database}}") +psw = conf_data.get("db_password", "{{db_password}}") +pghost = conf_data.get("db_host", "{{db_host}}") +pgport = 5432 +index_config = conf_data.get("index_config") +CONFIG = {} + +CONFIG["JSONIFY_PRETTYPRINT_REGULAR"] = False + +dist = environ.get("DIST", None) +if dist: + CONFIG["DIST"] = json.loads(dist) + +CONFIG["INDEX"] = { + "driver": SQLAlchemyIndexDriver( + "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( + usr=usr, + psw=psw, + pghost=pghost, + pgport=pgport, + db=db, + ), + index_config=index_config, + ), +} + +CONFIG["ALIAS"] = { + "driver": SQLAlchemyAliasDriver( + "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( + usr=usr, + psw=psw, + pghost=pghost, + pgport=pgport, + db=db, + ) + ), +} + +AUTH = SQLAlchemyAuthDriver( + "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( + usr=usr, + psw=psw, + pghost=pghost, + pgport=pgport, + db=db, + ), + arborist="http://arborist-service/", +) + +settings = {"config": CONFIG, "auth": AUTH} diff --git a/deployment/scripts/indexd/indexd_setup.sh b/deployment/scripts/indexd/indexd_setup.sh new file mode 100644 index 000000000..e6d02cd74 --- /dev/null +++ b/deployment/scripts/indexd/indexd_setup.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# entrypoint bash script for indexd + +## try to install python util, or else fallback to environment variables. + +# activate virtual env +pushd /indexd +# TODO: check if poetry can install globally +python -m venv py-venv && . py-venv/bin/activate && source $HOME/.poetry/env +poetry install -vv --no-dev --no-interaction && poetry show -v +python3 -m pip install cdispyutils +popd + +/dockerrun.sh \ No newline at end of file diff --git a/deployment/scripts/postgresql/postgresql_init.sql b/deployment/scripts/postgresql/postgresql_init.sql new file mode 100644 index 000000000..a276e3212 --- /dev/null +++ b/deployment/scripts/postgresql/postgresql_init.sql @@ -0,0 +1,34 @@ +/* Entrypoint script for postgresql container to set up databases and users +based on a script from: https://github.com/uc-cdis/compose-services/blob/master/scripts/postgres_init.sql + +See example usage in ADO: https://github.com/uc-cdis/indexd/pull/316/files#diff-1250fe52790be66c0718296d26be97fdaee7cfca04d0448f5b87dba907dfa669R42 + +This can also be used as part of a helm chart to setup the postgresql database (e.g. you can call this in from a shell script in a helm chart) +*/ + +CREATE DATABASE metadata_db; +CREATE DATABASE fence_db; +CREATE DATABASE indexd_db; +CREATE DATABASE arborist_db; + +CREATE USER fence_user; +ALTER USER fence_user WITH PASSWORD 'fence_pass'; +ALTER USER fence_user WITH SUPERUSER; + +CREATE USER peregrine_user; +ALTER USER peregrine_user WITH PASSWORD 'peregrine_pass'; +ALTER USER peregrine_user WITH SUPERUSER; + +CREATE USER sheepdog_user; +ALTER USER sheepdog_user WITH PASSWORD 'sheepdog_pass'; +ALTER USER sheepdog_user WITH SUPERUSER; + +CREATE USER indexd_user; +ALTER USER indexd_user WITH PASSWORD 'indexd_pass'; +ALTER USER indexd_user WITH SUPERUSER; + +CREATE USER arborist_user; +ALTER USER arborist_user WITH PASSWORD 'arborist_pass'; +ALTER USER arborist_user WITH SUPERUSER; + +ALTER USER postgres WITH PASSWORD 'postgres'; \ No newline at end of file diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini index 3d46220ad..0fa4f96a2 100644 --- a/deployment/uwsgi/uwsgi.ini +++ b/deployment/uwsgi/uwsgi.ini @@ -11,6 +11,10 @@ harakiri-verbose = true # No global HARAKIRI, using only user HARAKIRI, because export overwrites it # Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD # harakiri = 45 +; If VIRTUAL_ENV is set then use its value to specify the virtualenv directory +if-env = VIRTUAL_ENV +virtualenv = %(_) +endif = http-timeout = 45 socket-timeout = 45 worker-reload-mercy = 45 diff --git a/tests/default_test_settings.py b/tests/default_test_settings.py index df0e3e2ca..82583647a 100644 --- a/tests/default_test_settings.py +++ b/tests/default_test_settings.py @@ -1,33 +1,40 @@ -from indexd.default_settings import * -from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver -import os - -# override the default settings for INDEX because we want to test -# both PREPEND_PREFIX and ADD_PREFIX_ALIAS, which should not both -# be set to True in production environments -CONFIG["INDEX"] = { - "driver": SQLAlchemyIndexDriver( - "sqlite:///index.sq3", - auto_migrate=True, - echo=True, - index_config={ - "DEFAULT_PREFIX": "testprefix:", - "PREPEND_PREFIX": True, - "ADD_PREFIX_ALIAS": True, - }, - ) -} -CONFIG["DIST"] = [ - { - "name": "testStage", - "host": "https://fictitious-commons.io/index/", - "hints": [".*dg\\.4503.*"], - "type": "indexd", - } -] - -os.environ["PRESIGNED_FENCE_URL"] = "https://fictitious-commons.io/" -os.environ["HOSTNAME"] = "fictitious-commons.io" -settings = {"config": CONFIG, "auth": AUTH} - -settings["config"]["TEST_DB"] = "postgresql://postgres@localhost/test_migration_db" +import os + +from indexd.default_settings import * +from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver + + +# override the default settings for INDEX because we want to test +# both PREPEND_PREFIX and ADD_PREFIX_ALIAS, which should not both +# be set to True in production environments +CONFIG["INDEX"] = { + "driver": SQLAlchemyIndexDriver( + "sqlite:///index.sq3", + auto_migrate=True, + echo=True, + index_config={ + "DEFAULT_PREFIX": "testprefix:", + "PREPEND_PREFIX": True, + "ADD_PREFIX_ALIAS": True, + }, + ) +} +CONFIG["DIST"] = [ + { + "name": "testStage", + "host": "https://fictitious-commons.io/index/", + "hints": [".*dg\\.4503.*"], + "type": "indexd", + } +] + +os.environ["PRESIGNED_FENCE_URL"] = "https://fictitious-commons.io/" +os.environ["HOSTNAME"] = "fictitious-commons.io" +settings = {"config": CONFIG, "auth": AUTH} + +# Set PSQL Port, see https://www.postgresql.org/docs/12/app-psql.html +# PSQL default port is 5432, but in some setups, can be 5433. +psql_port = os.environ["PGPORT"] if os.environ.get("PGPORT") else "5432" +settings["config"][ + "TEST_DB" +] = "postgres://postgres:postgres@localhost:{0}/test_migration_db".format(psql_port)