From de21d098b763bc0c929777e491d3ffcb7b5dc8cb Mon Sep 17 00:00:00 2001 From: mibe Date: Thu, 3 Oct 2024 10:44:39 +0100 Subject: [PATCH] #70 Added LanguageContainerDeployerCli class --- .github/workflows/run-tests.yml | 2 +- .../cli/language_container_deployer_cli.py | 62 ++++++++++++ .../test_language_container_deployer_cli.py | 99 +++++++++++++++++++ test/integration/conftest.py | 32 +++--- test/unit/cli/test_std_options.py | 23 +++++ 5 files changed, 201 insertions(+), 17 deletions(-) create mode 100644 exasol/python_extension_common/cli/language_container_deployer_cli.py create mode 100644 test/integration/cli/test_language_container_deployer_cli.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f75447a..13edace 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -40,4 +40,4 @@ jobs: SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }} SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }} run: - poetry run nox -s integration-tests -- -- --db-version ${{ inputs.exasol-version}} --backend all + poetry run nox -s integration-tests -- -- --db-version ${{ inputs.exasol-version}} --backend onprem diff --git a/exasol/python_extension_common/cli/language_container_deployer_cli.py b/exasol/python_extension_common/cli/language_container_deployer_cli.py new file mode 100644 index 0000000..5a333a2 --- /dev/null +++ b/exasol/python_extension_common/cli/language_container_deployer_cli.py @@ -0,0 +1,62 @@ +from pathlib import Path + +from exasol.python_extension_common.deployment.language_container_deployer import LanguageContainerDeployer +from exasol.python_extension_common.connections.pyexasol_connection import open_pyexasol_connection +from exasol.python_extension_common.connections.bucketfs_location import create_bucketfs_location +from exasol.python_extension_common.cli.std_options import StdParams + + +class LanguageContainerDeployerCli: + """ + The class provides a CLI callback function that creates and runs the + LangaugeContainerDeployer. + + At first glance, it may look a bit over-designed. The reason for wrapping a + callback function in a class is to let the user define the option names for the + container URL and container name. These options are not defined in the StdParams + but rather generated by formatters. The user can give them arbitrary names. + Hence, we don't want to assume any particular names in the callback function. + """ + + def __init__(self, + container_url_arg: str | None = None, + container_name_arg: str | None = None) -> None: + self._container_url_arg = container_url_arg + self._container_name_arg = container_name_arg + + def __call__(self, **kwargs): + + pyexasol_connection = open_pyexasol_connection(**kwargs) + bucketfs_location = create_bucketfs_location(**kwargs) + + language_alias = kwargs[StdParams.language_alias.name] + container_file = kwargs[StdParams.container_file.name] + upload_container = kwargs[StdParams.upload_container.name] + alter_system = kwargs[StdParams.alter_system.name] + allow_override = kwargs[StdParams.allow_override.name] + wait_for_completion = kwargs[StdParams.wait_for_completion.name] + + deployer = LanguageContainerDeployer(pyexasol_connection, + language_alias, + bucketfs_location) + if not upload_container: + deployer.run(alter_system=alter_system, + allow_override=allow_override, + wait_for_completion=wait_for_completion) + elif container_file: + deployer.run(container_file=Path(container_file), + alter_system=alter_system, + allow_override=allow_override, + wait_for_completion=wait_for_completion) + elif (self._container_url_arg and self._container_name_arg and + kwargs[self._container_url_arg] and kwargs[self._container_name_arg]): + deployer.download_and_run(kwargs[self._container_url_arg], + kwargs[self._container_name_arg], + alter_system=alter_system, + allow_override=allow_override, + wait_for_completion=wait_for_completion) + else: + raise ValueError("To upload a language container either its release version " + f"(--{StdParams.version.name}) or a path of the already " + f"downloaded container file (--{StdParams.container_file.name}) " + "must be provided.") diff --git a/test/integration/cli/test_language_container_deployer_cli.py b/test/integration/cli/test_language_container_deployer_cli.py new file mode 100644 index 0000000..5dfdf30 --- /dev/null +++ b/test/integration/cli/test_language_container_deployer_cli.py @@ -0,0 +1,99 @@ +from typing import Any +from urllib.parse import urlparse +import pytest +import click +from click.testing import CliRunner + +from exasol.python_extension_common.cli.std_options import ( + StdTags, StdParams, ParameterFormatters, select_std_options) +from exasol.python_extension_common.connections.pyexasol_connection import ( + open_pyexasol_connection) +from exasol.python_extension_common.cli.language_container_deployer_cli import ( + LanguageContainerDeployerCli) +from test.utils.db_utils import assert_udf_running + +CONTAINER_URL_ARG = 'container_url' +CONTAINER_NAME_ARG = 'container_name' + + +@pytest.fixture +def onprem_params(exasol_config, + bucketfs_config, + language_alias) -> dict[str, Any]: + + parsed_url = urlparse(bucketfs_config.url) + host, port = parsed_url.netloc.split(":") + return { + StdParams.dsn.name: f'{exasol_config.host}:{exasol_config.port}', + StdParams.db_user.name: exasol_config.username, + StdParams.db_password.name: exasol_config.password, + StdParams.bucketfs_host.name: host, + StdParams.bucketfs_port.name: port, + StdParams.bucketfs_use_https.name: parsed_url.scheme.lower() == 'https', + StdParams.bucketfs_user.name: bucketfs_config.username, + StdParams.bucketfs_password.name: bucketfs_config.password, + StdParams.bucketfs_name.name: 'bfsdefault', + StdParams.bucket.name: 'default', + StdParams.use_ssl_cert_validation.name: False, + StdParams.path_in_bucket.name: 'container', + StdParams.language_alias: language_alias + } + + +def make_args_string(**kwargs) -> str: + return ' '.join(f'--{k} {v}' for k, v in kwargs.items()) + + +def test_slc_deployer_cli_onprem_url(use_onprem, + backend_aware_onprem_database, + container_version, + container_name, + container_url_formatter, + language_alias, + db_schema, + onprem_params): + if not use_onprem: + pytest.skip("The test is not configured to use ITDE.") + + ver_formatter = ParameterFormatters() + ver_formatter.set_formatter(CONTAINER_URL_ARG, container_url_formatter) + ver_formatter.set_formatter(CONTAINER_NAME_ARG, container_name) + + opts = select_std_options( + [StdTags.DB | StdTags.ONPREM, StdTags.BFS | StdTags.ONPREM, StdTags.SLC], + formatters={StdParams.version: ver_formatter}) + cli_callback = LanguageContainerDeployerCli( + container_url_arg=CONTAINER_URL_ARG, + container_name_arg=CONTAINER_NAME_ARG) + extra_params = {StdParams.version: container_version} + args = make_args_string(**onprem_params, **extra_params) + + cmd = click.Command('deploy_slc', params=opts, callback=cli_callback) + runner = CliRunner() + runner.invoke(cmd, args=args) + + with open_pyexasol_connection(**onprem_params) as conn: + assert_udf_running(conn, language_alias, db_schema) + + +def test_slc_deployer_cli_onprem_file(use_onprem, + backend_aware_onprem_database, + container_path, + language_alias, + db_schema, + onprem_params): + if not use_onprem: + pytest.skip("The test is not configured to use ITDE.") + + opts = select_std_options( + [StdTags.DB | StdTags.ONPREM, StdTags.BFS | StdTags.ONPREM, StdTags.SLC]) + cli_callback = LanguageContainerDeployerCli() + extra_params = {StdParams.container_file: container_path} + args = make_args_string(**onprem_params, **extra_params) + + cmd = click.Command('deploy_slc', params=opts, callback=cli_callback) + runner = CliRunner() + runner.invoke(cmd, args=args) + + with open_pyexasol_connection(**onprem_params) as conn: + assert_udf_running(conn, language_alias, db_schema) diff --git a/test/integration/conftest.py b/test/integration/conftest.py index 0cf352f..ec4b538 100644 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -14,27 +14,32 @@ from test.utils.revert_language_settings import revert_language_settings from test.utils.db_utils import create_schema, open_schema - -SLC_NAME = "template-Exasol-all-python-3.10_release.tar.gz" - -SLC_URL_FORMATTER = ("https://github.com/exasol/script-languages-release/releases/" - "download/{version}/") + SLC_NAME - VERSION = "8.0.0" TEST_SCHEMA = "PEC_DEPLOYER_TESTS" TEST_LANGUAGE_ALIAS = "PYTHON3_PEC_TESTS" +@pytest.fixture(scope='session') +def container_name() -> str: + return "template-Exasol-all-python-3.10_release.tar.gz" + + +@pytest.fixture(scope='session') +def container_url_formatter(container_name) -> str: + return ("https://github.com/exasol/script-languages-release/releases/" + "download/{version}/") + container_name + + @pytest.fixture -def main_func(): +def main_func(slc_name, slc_url_formatter): @click.group() def fake_main(): pass - slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, SLC_URL_FORMATTER) - slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, SLC_NAME) + slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, container_url_formatter) + slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, container_name) fake_main.add_command(language_container_deployer_main) return fake_main @@ -46,13 +51,8 @@ def container_version() -> str: @pytest.fixture(scope='session') -def container_name() -> str: - return SLC_NAME - - -@pytest.fixture(scope='session') -def container_url(container_version) -> str: - return SLC_URL_FORMATTER.format(version=VERSION) +def container_url(container_url_formatter, container_version) -> str: + return container_url_formatter.format(version=container_version) @pytest.fixture(scope='session') diff --git a/test/unit/cli/test_std_options.py b/test/unit/cli/test_std_options.py index f68ddef..8ae04c9 100644 --- a/test/unit/cli/test_std_options.py +++ b/test/unit/cli/test_std_options.py @@ -102,6 +102,29 @@ def test_select_std_options_with_override(): assert not opts[StdParams.alter_system.name].default +def test_select_std_options_with_formatter(): + container_url_arg = 'container_url' + container_name_arg = 'container_name' + url_format = "https://my_service_url/{version}/page" + name_format = "my_service_name" + version = '4.5.6' + expected_url = url_format.format(version=version) + expected_name = name_format + + def func(**kwargs): + assert kwargs[container_name_arg] == expected_name + assert kwargs[container_url_arg] == expected_url + + ver_formatter = ParameterFormatters() + ver_formatter.set_formatter(container_url_arg, url_format) + ver_formatter.set_formatter(container_name_arg, name_format) + + opts = select_std_options(StdTags.SLC, formatters={StdParams.version: ver_formatter}) + cmd = click.Command('do_something', params=opts, callback=func) + runner = CliRunner() + runner.invoke(cmd, args=f'--version {version}') + + def test_hidden_opt_with_envar(): """ This test checks the mechanism of providing a value of a confidential parameter