From 17f44ac919431da7b503a84f941364250655e006 Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 26 Mar 2024 09:20:50 +0000 Subject: [PATCH 1/4] Add documentation build folder to .gitignore --- .gitignore | 1 + {changes => doc/changes}/changelog.md | 0 {changes => doc/changes}/changes_0.1.0.md | 0 exasol/python_extension_common/__init__.py | 0 .../deployment/language_container_deployer.py | 291 ++++++++++++++++++ .../language_container_deployer_cli.py | 168 ++++++++++ pyproject.toml | 15 + .../test_language_container_deployer.py | 133 ++++++++ .../test_language_container_deployer_cli.py | 29 ++ 9 files changed, 637 insertions(+) create mode 100644 .gitignore rename {changes => doc/changes}/changelog.md (100%) rename {changes => doc/changes}/changes_0.1.0.md (100%) create mode 100644 exasol/python_extension_common/__init__.py create mode 100644 exasol/python_extension_common/deployment/language_container_deployer.py create mode 100644 exasol/python_extension_common/deployment/language_container_deployer_cli.py create mode 100644 pyproject.toml create mode 100644 test/unit/deployment/test_language_container_deployer.py create mode 100644 test/unit/deployment/test_language_container_deployer_cli.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8568120 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.html-documentation diff --git a/changes/changelog.md b/doc/changes/changelog.md similarity index 100% rename from changes/changelog.md rename to doc/changes/changelog.md diff --git a/changes/changes_0.1.0.md b/doc/changes/changes_0.1.0.md similarity index 100% rename from changes/changes_0.1.0.md rename to doc/changes/changes_0.1.0.md diff --git a/exasol/python_extension_common/__init__.py b/exasol/python_extension_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exasol/python_extension_common/deployment/language_container_deployer.py b/exasol/python_extension_common/deployment/language_container_deployer.py new file mode 100644 index 0000000..c260965 --- /dev/null +++ b/exasol/python_extension_common/deployment/language_container_deployer.py @@ -0,0 +1,291 @@ +from enum import Enum +from textwrap import dedent +from typing import List, Optional +from pathlib import Path, PurePosixPath +import logging +import tempfile +import requests +import ssl +import pyexasol +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from exasol_bucketfs_utils_python.bucket_config import BucketConfig, BucketFSConfig +from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig + +logger = logging.getLogger(__name__) + + +def create_bucketfs_location( + bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, + bucketfs_use_https: bool, bucketfs_user: str, bucketfs_password: str, + bucket: str, path_in_bucket: str) -> BucketFSLocation: + _bucketfs_connection = BucketFSConnectionConfig( + host=bucketfs_host, port=bucketfs_port, user=bucketfs_user, + pwd=bucketfs_password, is_https=bucketfs_use_https) + _bucketfs_config = BucketFSConfig( + bucketfs_name=bucketfs_name, connection_config=_bucketfs_connection) + _bucket_config = BucketConfig( + bucket_name=bucket, bucketfs_config=_bucketfs_config) + return BucketFSLocation( + bucket_config=_bucket_config, + base_path=PurePosixPath(path_in_bucket)) + + +def get_websocket_sslopt(use_ssl_cert_validation: bool = True, + ssl_trusted_ca: Optional[str] = None, + ssl_client_certificate: Optional[str] = None, + ssl_private_key: Optional[str] = None) -> dict: + """ + Returns a dictionary in the winsocket-client format + (see https://websocket-client.readthedocs.io/en/latest/faq.html#what-else-can-i-do-with-sslopts) + """ + + # Is server certificate validation required? + sslopt: dict[str, object] = {"cert_reqs": ssl.CERT_REQUIRED if use_ssl_cert_validation else ssl.CERT_NONE} + + # Is a bundle with trusted CAs provided? + if ssl_trusted_ca: + trusted_ca_path = Path(ssl_trusted_ca) + if trusted_ca_path.is_dir(): + sslopt["ca_cert_path"] = ssl_trusted_ca + elif trusted_ca_path.is_file(): + sslopt["ca_certs"] = ssl_trusted_ca + else: + raise ValueError(f"Trusted CA location {ssl_trusted_ca} doesn't exist.") + + # Is client's own certificate provided? + if ssl_client_certificate: + if not Path(ssl_client_certificate).is_file(): + raise ValueError(f"Certificate file {ssl_client_certificate} doesn't exist.") + sslopt["certfile"] = ssl_client_certificate + if ssl_private_key: + if not Path(ssl_private_key).is_file(): + raise ValueError(f"Private key file {ssl_private_key} doesn't exist.") + sslopt["keyfile"] = ssl_private_key + + return sslopt + + +class LanguageActivationLevel(Enum): + f""" + Language activation level, i.e. + ALTER SET SCRIPT_LANGUAGES=... + """ + Session = 'SESSION' + System = 'SYSTEM' + + +def get_language_settings(pyexasol_conn: pyexasol.ExaConnection, alter_type: LanguageActivationLevel) -> str: + """ + Reads the current language settings at the specified level. + + pyexasol_conn - Opened database connection. + alter_type - Activation level - SYSTEM or SESSION. + """ + result = pyexasol_conn.execute( + f"""SELECT "{alter_type.value}_VALUE" FROM SYS.EXA_PARAMETERS WHERE + PARAMETER_NAME='SCRIPT_LANGUAGES'""").fetchall() + return result[0][0] + + +class LanguageContainerDeployer: + + def __init__(self, + pyexasol_connection: pyexasol.ExaConnection, + language_alias: str, + bucketfs_location: BucketFSLocation) -> None: + + self._bucketfs_location = bucketfs_location + self._language_alias = language_alias + self._pyexasol_conn = pyexasol_connection + logger.debug(f"Init {LanguageContainerDeployer.__name__}") + + def download_and_run(self, url: str, + bucket_file_path: str, + alter_system: bool = True, + allow_override: bool = False) -> None: + """ + Downloads the language container from the provided url to a temporary file and then deploys it. + See docstring on the `run` method for details on what is involved in the deployment. + + url - Address where the container will be downloaded from. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + alter_system - If True will try to activate the container at the System level. + allow_override - If True the activation of a language container with the same alias will be + overriden, otherwise a RuntimeException will be thrown. + """ + + with tempfile.NamedTemporaryFile() as tmp_file: + response = requests.get(url, stream=True) + response.raise_for_status() + tmp_file.write(response.content) + + self.run(Path(tmp_file.name), bucket_file_path, alter_system, allow_override) + + def run(self, container_file: Optional[Path] = None, + bucket_file_path: Optional[str] = None, + alter_system: bool = True, + allow_override: bool = False) -> None: + """ + Deploys the language container. This includes two steps, both of which are optional: + - Uploading the container into the database. This step can be skipped if the container + has already been uploaded. + - Activating the container. This step may have to be skipped if the user does not have + System Privileges in the database. In that case two alternative activation SQL commands + will be printed on the console. + + container_file - Path of the container tar.gz file in a local file system. + If not provided the container is assumed to be uploaded already. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + If not specified the name of the container file will be used instead. + alter_system - If True will try to activate the container at the System level. + allow_override - If True the activation of a language container with the same alias will be + overriden, otherwise a RuntimeException will be thrown. + """ + + if not bucket_file_path: + if not container_file: + raise ValueError('Either a container file or a bucket file path must be specified.') + bucket_file_path = container_file.name + + if container_file: + self.upload_container(container_file, bucket_file_path) + + if alter_system: + self.activate_container(bucket_file_path, LanguageActivationLevel.System, allow_override) + else: + message = dedent(f""" + In SQL, you can activate the SLC of the Transformers Extension + by using the following statements: + + To activate the SLC only for the current session: + {self.generate_activation_command(bucket_file_path, LanguageActivationLevel.Session, True)} + + To activate the SLC on the system: + {self.generate_activation_command(bucket_file_path, LanguageActivationLevel.System, True)} + """) + print(message) + + def upload_container(self, container_file: Path, + bucket_file_path: Optional[str] = None) -> None: + """ + Upload the language container to the BucketFS. + + container_file - Path of the container tar.gz file in a local file system. + bucket_file_path - Path within the designated bucket where the container should be uploaded. + """ + if not container_file.is_file(): + raise RuntimeError(f"Container file {container_file} " + f"is not a file.") + with open(container_file, "br") as f: + self._bucketfs_location.upload_fileobj_to_bucketfs( + fileobj=f, bucket_file_path=bucket_file_path) + logging.debug("Container is uploaded to bucketfs") + + def activate_container(self, bucket_file_path: str, + alter_type: LanguageActivationLevel = LanguageActivationLevel.Session, + allow_override: bool = False) -> None: + """ + Activates the language container at the required level. + + bucket_file_path - Path within the designated bucket where the container is uploaded. + alter_type - Language activation level, defaults to the SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + """ + alter_command = self.generate_activation_command(bucket_file_path, alter_type, allow_override) + self._pyexasol_conn.execute(alter_command) + logging.debug(alter_command) + + def generate_activation_command(self, bucket_file_path: str, + alter_type: LanguageActivationLevel, + allow_override: bool = False) -> str: + """ + Generates an SQL command to activate the SLC container at the required level. The command will + preserve existing activations of other containers identified by different language aliases. + Activation of a container with the same alias, if exists, will be overwritten. + + bucket_file_path - Path within the designated bucket where the container is uploaded. + alter_type - Activation level - SYSTEM or SESSION. + allow_override - If True the activation of a language container with the same alias will be overriden, + otherwise a RuntimeException will be thrown. + """ + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path) + new_settings = \ + self._update_previous_language_settings(alter_type, allow_override, path_in_udf) + alter_command = \ + f"ALTER {alter_type.value} SET SCRIPT_LANGUAGES='{new_settings}';" + return alter_command + + def _update_previous_language_settings(self, alter_type: LanguageActivationLevel, + allow_override: bool, + path_in_udf: PurePosixPath) -> str: + prev_lang_settings = get_language_settings(self._pyexasol_conn, alter_type) + prev_lang_aliases = prev_lang_settings.split(" ") + self._check_if_requested_language_alias_already_exists( + allow_override, prev_lang_aliases) + new_definitions_str = self._generate_new_language_settings( + path_in_udf, prev_lang_aliases) + return new_definitions_str + + def get_language_definition(self, bucket_file_path: str): + """ + Generate a language definition (ALIAS=URL) for the specified bucket file path. + + bucket_file_path - Path within the designated bucket where the container is uploaded. + """ + path_in_udf = self._bucketfs_location.generate_bucket_udf_path(bucket_file_path) + result = self._generate_new_language_settings(path_in_udf=path_in_udf, prev_lang_aliases=[]) + return result + + def _generate_new_language_settings(self, path_in_udf: PurePosixPath, + prev_lang_aliases: List[str]) -> str: + other_definitions = [ + alias_definition for alias_definition in prev_lang_aliases + if not alias_definition.startswith(self._language_alias + "=")] + path_in_udf_without_buckets = PurePosixPath(*path_in_udf.parts[2:]) + new_language_alias_definition = \ + f"{self._language_alias}=localzmq+protobuf:///" \ + f"{path_in_udf_without_buckets}?lang=python#" \ + f"{path_in_udf}/exaudf/exaudfclient_py3" + new_definitions = other_definitions + [new_language_alias_definition] + new_definitions_str = " ".join(new_definitions) + return new_definitions_str + + def _check_if_requested_language_alias_already_exists( + self, allow_override: bool, + prev_lang_aliases: List[str]) -> None: + definition_for_requested_alias = [ + alias_definition for alias_definition in prev_lang_aliases + if alias_definition.startswith(self._language_alias + "=")] + if not len(definition_for_requested_alias) == 0: + warning_message = f"The requested language alias {self._language_alias} is already in use." + if allow_override: + logging.warning(warning_message) + else: + raise RuntimeError(warning_message) + + @classmethod + def create(cls, bucketfs_name: str, bucketfs_host: str, bucketfs_port: int, + bucketfs_use_https: bool, bucketfs_user: str, + bucketfs_password: str, bucket: str, path_in_bucket: str, + dsn: str, db_user: str, db_password: str, language_alias: str, + use_ssl_cert_validation: bool = True, ssl_trusted_ca: Optional[str] = None, + ssl_client_certificate: Optional[str] = None, + ssl_private_key: Optional[str] = None) -> "LanguageContainerDeployer": + + websocket_sslopt = get_websocket_sslopt(use_ssl_cert_validation, ssl_trusted_ca, + ssl_client_certificate, ssl_private_key) + + pyexasol_conn = pyexasol.connect( + dsn=dsn, + user=db_user, + password=db_password, + encryption=True, + websocket_sslopt=websocket_sslopt + ) + + bucketfs_location = create_bucketfs_location( + bucketfs_name, bucketfs_host, bucketfs_port, bucketfs_use_https, + bucketfs_user, bucketfs_password, bucket, path_in_bucket) + + return cls(pyexasol_conn, language_alias, bucketfs_location) diff --git a/exasol/python_extension_common/deployment/language_container_deployer_cli.py b/exasol/python_extension_common/deployment/language_container_deployer_cli.py new file mode 100644 index 0000000..fb0b222 --- /dev/null +++ b/exasol/python_extension_common/deployment/language_container_deployer_cli.py @@ -0,0 +1,168 @@ +from typing import Optional, Any +import os +import re +import click +from enum import Enum +from pathlib import Path +from exasol.python_transformers_extension.deployment import deployment_utils as utils +from exasol_transformers_extension.deployment.language_container_deployer import LanguageContainerDeployer + + +class CustomizableParameters(Enum): + """ + Parameters of the cli that can be programmatically customised by a developer + of a specialised version of the cli. + The names in the enum list should match the parameter names in language_container_deployer_main. + """ + container_url = 1 + container_name = 2 + + +class _ParameterFormatters: + """ + Class facilitating customization of the cli. + + The idea is that some of the cli parameters can be programmatically customized based + on values of other parameters and externally supplied formatters. For example a specialized + version of the cli may want to provide its own url. Furthermore, this url will depend on + the user supplied parameter called "version". The solution is to set a formatter for the + url, for instance "http://my_stuff/{version}/my_data". If the user specifies non-empty version + parameter the url will be fully formed. + + A formatter may include more than one parameter. In the previous example the url could, + for instance, also include a username: "http://my_stuff/{version}/{user}/my_data". + + Note that customized parameters can only be updated in a callback function. There is no + way to inject them directly into the cli. Also, the current implementation doesn't perform + the update if the value of the parameter dressed with the callback is None. + + IMPORTANT! Please make sure that the formatters are set up before the call to the cli function, + e.g. language_container_deployer_main, is executed. + """ + def __init__(self): + self._formatters = {} + + def __call__(self, ctx: click.Context, param: click.Parameter, value: Optional[Any]) -> Optional[Any]: + + def update_parameter(parameter_name: str, formatter: str) -> None: + param_formatter = ctx.params.get(parameter_name, formatter) + if param_formatter: + # Enclose in double curly brackets all other parameters in the formatting string, + # to avoid the missing parameters' error. Below is an example of a formatter string + # before and after applying the regex, assuming the current parameter is 'version'. + # 'something-with-{version}/tailored-for-{user}' => 'something-with-{version}/tailored-for-{{user}}' + # We were looking for all occurrences of a pattern '{some_name}', where some_name is not version. + pattern = r'\{(?!' + param.name + r'\})\w+\}' + param_formatter = re.sub(pattern, lambda m: f'{{{m.group(0)}}}', param_formatter) + kwargs = {param.name: value} + ctx.params[parameter_name] = param_formatter.format(**kwargs) + + if value is not None: + for prm_name, prm_formatter in self._formatters.items(): + update_parameter(prm_name, prm_formatter) + + return value + + def set_formatter(self, custom_parameter: CustomizableParameters, formatter: str) -> None: + """ Sets a formatter for a customizable parameter. """ + self._formatters[custom_parameter.name] = formatter + + def clear_formatters(self): + """ Deletes all formatters, mainly for testing purposes. """ + self._formatters.clear() + + +# Global cli customization object. +# Specialized versions of this cli should use this object to set custom parameter formatters. +slc_parameter_formatters = _ParameterFormatters() + + +@click.command(name="language-container") +@click.option('--bucketfs-name', type=str, required=True) +@click.option('--bucketfs-host', type=str, required=True) +@click.option('--bucketfs-port', type=int, required=True) +@click.option('--bucketfs-use-https', type=bool, default=False) +@click.option('--bucketfs-user', type=str, required=True, default="w") +@click.option('--bucketfs-password', prompt='bucketFS password', hide_input=True, + default=lambda: os.environ.get(utils.BUCKETFS_PASSWORD_ENVIRONMENT_VARIABLE, "")) +@click.option('--bucket', type=str, required=True) +@click.option('--path-in-bucket', type=str, required=True, default=None) +@click.option('--container-file', + type=click.Path(exists=True, file_okay=True), default=None) +@click.option('--version', type=str, default=None, expose_value=False, + callback=slc_parameter_formatters) +@click.option('--dsn', type=str, required=True) +@click.option('--db-user', type=str, required=True) +@click.option('--db-pass', prompt='db password', hide_input=True, + default=lambda: os.environ.get(utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, "")) +@click.option('--language-alias', type=str, default="PYTHON3_TE") +@click.option('--ssl-cert-path', type=str, default="") +@click.option('--ssl-client-cert-path', type=str, default="") +@click.option('--ssl-client-private-key', type=str, default="") +@click.option('--use-ssl-cert-validation/--no-use-ssl-cert-validation', type=bool, default=True) +@click.option('--upload-container/--no-upload_container', type=bool, default=True) +@click.option('--alter-system/--no-alter-system', type=bool, default=True) +@click.option('--allow-override/--disallow-override', type=bool, default=False) +def language_container_deployer_main( + bucketfs_name: str, + bucketfs_host: str, + bucketfs_port: int, + bucketfs_use_https: bool, + bucketfs_user: str, + bucketfs_password: str, + bucket: str, + path_in_bucket: str, + container_file: str, + dsn: str, + db_user: str, + db_pass: str, + language_alias: str, + ssl_cert_path: str, + ssl_client_cert_path: str, + ssl_client_private_key: str, + use_ssl_cert_validation: bool, + upload_container: bool, + alter_system: bool, + allow_override: bool, + container_url: str = None, + container_name: str = None): + + deployer = LanguageContainerDeployer.create( + bucketfs_name=bucketfs_name, + bucketfs_host=bucketfs_host, + bucketfs_port=bucketfs_port, + bucketfs_use_https=bucketfs_use_https, + bucketfs_user=bucketfs_user, + bucketfs_password=bucketfs_password, + bucket=bucket, + path_in_bucket=path_in_bucket, + dsn=dsn, + db_user=db_user, + db_password=db_pass, + language_alias=language_alias, + ssl_trusted_ca=ssl_cert_path, + ssl_client_certificate=ssl_client_cert_path, + ssl_private_key=ssl_client_private_key, + use_ssl_cert_validation=use_ssl_cert_validation) + + if not upload_container: + deployer.run(alter_system=alter_system, allow_override=allow_override) + elif container_file: + deployer.run(container_file=Path(container_file), alter_system=alter_system, allow_override=allow_override) + elif container_url and container_name: + deployer.download_and_run(container_url, container_name, alter_system=alter_system, + allow_override=allow_override) + else: + # The error message should mention the parameters which the callback is specified for being missed. + raise ValueError("To upload a language container you should specify either its " + "release version or a path of the already downloaded container file.") + + +if __name__ == '__main__': + import logging + + logging.basicConfig( + format='%(asctime)s - %(module)s - %(message)s', + level=logging.DEBUG) + + language_container_deployer_main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..aa8c864 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.poetry] +name = "python-extension-common" +version = "0.1.0" +description = "A collection of common utilities for Exasol extensions." +authors = ["Your Name "] +license = "MIT" +readme = "README.md" + +[tool.poetry.dependencies] +python = "[^3.8]" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/test/unit/deployment/test_language_container_deployer.py b/test/unit/deployment/test_language_container_deployer.py new file mode 100644 index 0000000..746c442 --- /dev/null +++ b/test/unit/deployment/test_language_container_deployer.py @@ -0,0 +1,133 @@ +######################################################### +# To be migrated to the script-languages-container-tool # +######################################################### +from pathlib import Path, PurePosixPath +from unittest.mock import create_autospec, MagicMock, patch + +import pytest +from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation +from pyexasol import ExaConnection + +from exasol_transformers_extension.deployment.language_container_deployer import ( + LanguageContainerDeployer, LanguageActivationLevel) + + +@pytest.fixture(scope='module') +def container_file_name() -> str: + return 'container_xyz.tag.gz' + + +@pytest.fixture(scope='module') +def container_file_path(container_file_name) -> Path: + return Path(container_file_name) + + +@pytest.fixture(scope='module') +def language_alias() -> str: + return 'PYTHON3_TEST' + + +@pytest.fixture(scope='module') +def container_bfs_path(container_file_name) -> str: + return f'bfsdefault/default/container/{container_file_name[:-7]}' + + +@pytest.fixture(scope='module') +def mock_pyexasol_conn() -> ExaConnection: + return create_autospec(ExaConnection) + + +@pytest.fixture(scope='module') +def mock_bfs_location(container_bfs_path) -> BucketFSLocation: + mock_loc = create_autospec(BucketFSLocation) + mock_loc.generate_bucket_udf_path.return_value = PurePosixPath(f'/buckets/{container_bfs_path}') + return mock_loc + + +@pytest.fixture +def container_deployer(mock_pyexasol_conn, mock_bfs_location, language_alias) -> LanguageContainerDeployer: + deployer = LanguageContainerDeployer(pyexasol_connection=mock_pyexasol_conn, + language_alias=language_alias, + bucketfs_location=mock_bfs_location) + + deployer.upload_container = MagicMock() + deployer.activate_container = MagicMock() + return deployer + + +def test_slc_deployer_deploy(container_deployer, container_file_name, container_file_path): + container_deployer.run(container_file=container_file_path, bucket_file_path=container_file_name, alter_system=True, + allow_override=True) + container_deployer.upload_container.assert_called_once_with(container_file_path, container_file_name) + container_deployer.activate_container.assert_called_once_with(container_file_name, LanguageActivationLevel.System, + True) + + +def test_slc_deployer_upload(container_deployer, container_file_name, container_file_path): + container_deployer.run(container_file=container_file_path, alter_system=False) + container_deployer.upload_container.assert_called_once_with(container_file_path, container_file_name) + container_deployer.activate_container.assert_not_called() + + +def test_slc_deployer_activate(container_deployer, container_file_name, container_file_path): + container_deployer.run(bucket_file_path=container_file_name, alter_system=True, allow_override=True) + container_deployer.upload_container.assert_not_called() + container_deployer.activate_container.assert_called_once_with(container_file_name, LanguageActivationLevel.System, + True) + + +@patch('exasol_transformers_extension.deployment.language_container_deployer.get_language_settings') +def test_slc_deployer_generate_activation_command(mock_lang_settings, container_deployer, language_alias, + container_file_name, container_bfs_path): + mock_lang_settings.return_value = 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3' + + alter_type = LanguageActivationLevel.Session + expected_command = f"ALTER {alter_type.value.upper()} SET SCRIPT_LANGUAGES='" \ + "R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 " \ + f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3';" + + command = container_deployer.generate_activation_command(container_file_name, alter_type) + assert command == expected_command + + +@patch('exasol_transformers_extension.deployment.language_container_deployer.get_language_settings') +def test_slc_deployer_generate_activation_command_override(mock_lang_settings, container_deployer, language_alias, + container_file_name, container_bfs_path): + current_bfs_path = 'bfsdefault/default/container_abc' + mock_lang_settings.return_value = \ + 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 ' \ + f'{language_alias}=localzmq+protobuf:///{current_bfs_path}?' \ + f'lang=python#/buckets/{current_bfs_path}/exaudf/exaudfclient_py3' + + alter_type = LanguageActivationLevel.Session + expected_command = f"ALTER {alter_type.value.upper()} SET SCRIPT_LANGUAGES='" \ + "R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 " \ + f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3';" + + command = container_deployer.generate_activation_command(container_file_name, alter_type, allow_override=True) + assert command == expected_command + + +@patch('exasol_transformers_extension.deployment.language_container_deployer.get_language_settings') +def test_slc_deployer_generate_activation_command_failure(mock_lang_settings, container_deployer, language_alias, + container_file_name): + current_bfs_path = 'bfsdefault/default/container_abc' + mock_lang_settings.return_value = \ + 'R=builtin_r JAVA=builtin_java PYTHON3=builtin_python3 ' \ + f'{language_alias}=localzmq+protobuf:///{current_bfs_path}?' \ + f'lang=python#/buckets/{current_bfs_path}/exaudf/exaudfclient_py3' + + with pytest.raises(RuntimeError): + container_deployer.generate_activation_command(container_file_name, LanguageActivationLevel.Session, + allow_override=False) + + +def test_slc_deployer_get_language_definition(container_deployer, language_alias, + container_file_name, container_bfs_path): + expected_command = f"{language_alias}=localzmq+protobuf:///{container_bfs_path}?" \ + f"lang=python#/buckets/{container_bfs_path}/exaudf/exaudfclient_py3" + + command = container_deployer.get_language_definition(container_file_name) + assert command == expected_command diff --git a/test/unit/deployment/test_language_container_deployer_cli.py b/test/unit/deployment/test_language_container_deployer_cli.py new file mode 100644 index 0000000..dcaf837 --- /dev/null +++ b/test/unit/deployment/test_language_container_deployer_cli.py @@ -0,0 +1,29 @@ +import click +from exasol_transformers_extension.deployment.language_container_deployer_cli import ( + _ParameterFormatters, CustomizableParameters) + + +def test_parameter_formatters_1param(): + cmd = click.Command('a_command') + ctx = click.Context(cmd) + opt = click.Option(['--version']) + formatters = _ParameterFormatters() + formatters.set_formatter(CustomizableParameters.container_url, 'http://my_server/{version}/my_stuff') + formatters.set_formatter(CustomizableParameters.container_name, 'downloaded') + formatters(ctx, opt, '1.3.2') + assert ctx.params[CustomizableParameters.container_url.name] == 'http://my_server/1.3.2/my_stuff' + assert ctx.params[CustomizableParameters.container_name.name] == 'downloaded' + + +def test_parameter_formatters_2params(): + cmd = click.Command('a_command') + ctx = click.Context(cmd) + opt1 = click.Option(['--version']) + opt2 = click.Option(['--user']) + formatters = _ParameterFormatters() + formatters.set_formatter(CustomizableParameters.container_url, 'http://my_server/{version}/{user}/my_stuff') + formatters.set_formatter(CustomizableParameters.container_name, 'downloaded-{version}') + formatters(ctx, opt1, '1.3.2') + formatters(ctx, opt2, 'cezar') + assert ctx.params[CustomizableParameters.container_url.name] == 'http://my_server/1.3.2/cezar/my_stuff' + assert ctx.params[CustomizableParameters.container_name.name] == 'downloaded-1.3.2' From 3865500160a726e676174cab1651b280678c4698 Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 1 Oct 2024 12:28:40 +0100 Subject: [PATCH 2/4] #66 Standard CLI command builder --- doc/changes/unreleased.md | 4 + .../cli/std_options.py | 238 ++++++++++++++++++ test/unit/cli/test_std_options.py | 122 +++++++++ 3 files changed, 364 insertions(+) create mode 100644 exasol/python_extension_common/cli/std_options.py create mode 100644 test/unit/cli/test_std_options.py diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 79e701b..51beab6 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1 +1,5 @@ # Unreleased + +## Features + +* #66: Implement a standard CLI command builder. diff --git a/exasol/python_extension_common/cli/std_options.py b/exasol/python_extension_common/cli/std_options.py new file mode 100644 index 0000000..741b797 --- /dev/null +++ b/exasol/python_extension_common/cli/std_options.py @@ -0,0 +1,238 @@ +from typing import Any, no_type_check +import os +import re +from enum import Flag, Enum, auto +import click + + +class ParameterFormatters: + """ + Class facilitating customization of the cli. + + The idea is that some of the cli parameters can be programmatically customized based + on values of other parameters and externally supplied formatters. For example a specialized + version of the cli may want to provide its own url. Furthermore, this url will depend on + the user supplied parameter called "version". The solution is to set a formatter for the + url, for instance "http://my_stuff/{version}/my_data". If the user specifies non-empty version + parameter the url will be fully formed. + + A formatter may include more than one parameter. In the previous example the url could, + for instance, also include a username: "http://my_stuff/{version}/{user}/my_data". + + Note that customized parameters can only be updated in a callback function. There is no + way to inject them directly into the cli. Also, the current implementation doesn't perform + the update if the value of the parameter dressed with the callback is None. + """ + def __init__(self): + self._formatters = {} + + def __call__(self, ctx: click.Context, param: click.Parameter, value: Any | None) -> Any | None: + + def update_parameter(parameter_name: str, formatter: str) -> None: + param_formatter = ctx.params.get(parameter_name, formatter) + if param_formatter: + # Enclose in double curly brackets all other parameters in the formatting string, + # to avoid the missing parameters' error. Below is an example of a formatter string + # before and after applying the regex, assuming the current parameter is 'version'. + # 'something-with-{version}/tailored-for-{user}' => 'something-with-{version}/tailored-for-{{user}}' + # We were looking for all occurrences of a pattern '{some_name}', where some_name is not version. + pattern = r'\{(?!' + (param.name or '') + r'\})\w+\}' + param_formatter = re.sub(pattern, lambda m: f'{{{m.group(0)}}}', param_formatter) + kwargs = {param.name: value} + ctx.params[parameter_name] = param_formatter.format(**kwargs) + + if value is not None: + for prm_name, prm_formatter in self._formatters.items(): + update_parameter(prm_name, prm_formatter) + + return value + + def set_formatter(self, custom_parameter_name: str, formatter: str) -> None: + """ Sets a formatter for a customizable parameter. """ + self._formatters[custom_parameter_name] = formatter + + def clear_formatters(self): + """ Deletes all formatters, mainly for testing purposes. """ + self._formatters.clear() + + +# This text will be displayed instead of the actual value for a "secret" option. +SECRET_DISPLAY = '***' + + +def secret_callback(ctx: click.Context, param: click.Option, value: Any): + """ + Here we try to get the secret option value from an environment variable. + The reason for doing this in the callback instead of using a callable default is + that we don't want the default to be displayed in the prompt. There seems to + be no way of altering this behaviour. + """ + if value == SECRET_DISPLAY: + envar_name = param.opts[0][2:].upper() + return os.environ.get(envar_name) + return value + + +class StdTags(Flag): + DB = auto() + BFS = auto() + ONPREM = auto() + SAAS = auto() + SLC = auto() + + +class StdParams(Enum): + """ + Standard option keys. + """ + bucketfs_name = (StdTags.BFS | StdTags.ONPREM, auto()) + bucketfs_host = (StdTags.BFS | StdTags.ONPREM, auto()) + bucketfs_port = (StdTags.BFS | StdTags.ONPREM, auto()) + bucketfs_use_https = (StdTags.BFS | StdTags.ONPREM, auto()) + bucketfs_user = (StdTags.BFS | StdTags.ONPREM, auto()) + bucketfs_password = (StdTags.BFS | StdTags.ONPREM, auto()) + bucket = (StdTags.BFS | StdTags.ONPREM, auto()) + saas_url = (StdTags.DB | StdTags.BFS | StdTags.SAAS, auto()) + saas_account_id = (StdTags.DB | StdTags.BFS | StdTags.SAAS, auto()) + saas_database_id = (StdTags.DB | StdTags.BFS | StdTags.SAAS, auto()) + saas_database_name = (StdTags.DB | StdTags.BFS | StdTags.SAAS, auto()) + saas_token = (StdTags.DB | StdTags.BFS | StdTags.SAAS, auto()) + path_in_bucket = (StdTags.BFS | StdTags.ONPREM | StdTags.SAAS, auto()) + container_file = (StdTags.SLC, auto()) + version = (StdTags.SLC, auto()) + dsn = (StdTags.DB | StdTags.ONPREM, auto()) + db_user = (StdTags.DB | StdTags.ONPREM, auto()) + db_password = (StdTags.DB | StdTags.ONPREM, auto()) + language_alias = (StdTags.SLC, auto()) + schema = (StdTags.DB | StdTags.ONPREM | StdTags.SAAS, auto()) + ssl_cert_path = (StdTags.DB | StdTags.ONPREM, auto()) + ssl_client_cert_path = (StdTags.DB | StdTags.ONPREM, auto()) + ssl_client_private_key = (StdTags.DB | StdTags.ONPREM, auto()) + use_ssl_cert_validation = (StdTags.DB | StdTags.BFS | StdTags.ONPREM, auto()) + upload_container = (StdTags.SLC, auto()) + alter_system = (StdTags.SLC, auto()) + allow_override = (StdTags.SLC, auto()) + wait_for_completion = (StdTags.SLC, auto()) + + def __init__(self, tags: StdTags, value): + self.tags = tags + + +""" +Standard options defined in the form of key-value pairs, where key is the option's +StaParam key and the value is a kwargs for creating the click.Options(...). +""" +_std_options = { + StdParams.bucketfs_name: {'type': str}, + StdParams.bucketfs_host: {'type': str}, + StdParams.bucketfs_port: {'type': int}, + StdParams.bucketfs_use_https: {'type': bool, 'default': False}, + StdParams.bucketfs_user: {'type': str}, + StdParams.bucketfs_password: {'type': str, 'hide_input': True}, + StdParams.bucket: {'type': str}, + StdParams.saas_url: {'type': str, 'default': 'https://cloud.exasol.com'}, + StdParams.saas_account_id: {'type': str, 'hide_input': True}, + StdParams.saas_database_id: {'type': str, 'hide_input': True}, + StdParams.saas_database_name: {'type': str}, + StdParams.saas_token: {'type': str, 'hide_input': True}, + StdParams.path_in_bucket: {'type': str}, + StdParams.container_file: {'type': click.Path(exists=True, file_okay=True)}, + StdParams.version: {'type': str, 'expose_value': False}, + StdParams.dsn: {'type': str}, + StdParams.db_user: {'type': str}, + StdParams.db_password: {'type': str, 'hide_input': True}, + StdParams.language_alias: {'type': str}, + StdParams.schema: {'type': str, 'default': ''}, + StdParams.ssl_cert_path: {'type': str, 'default': ''}, + StdParams.ssl_client_cert_path: {'type': str, 'default': ''}, + StdParams.ssl_client_private_key: {'type': str, 'default': ''}, + StdParams.use_ssl_cert_validation: {'type': bool, 'default': True}, + StdParams.upload_container: {'type': bool, 'default': True}, + StdParams.alter_system: {'type': bool, 'default': True}, + StdParams.allow_override: {'type': bool, 'default': False}, + StdParams.wait_for_completion: {'type': bool, 'default': True} +} + + +def make_option_secret(option_params: dict[str, Any], prompt: str) -> None: + """ + Makes an option "secret" in the way that its input is not leaked to the + terminal. The option can be either a standard or a user defined. + + Parameters: + option_params: + Option properties. + prompt: + The prompt text for this option. + """ + option_params['hide_input'] = True + option_params['prompt'] = prompt + option_params['prompt_required'] = False + option_params['default'] = SECRET_DISPLAY + option_params['callback'] = secret_callback + + +def create_std_option(std_param: StdParams, **kwargs) -> click.Option: + """ + Creates a Click option. + + Parameters: + std_param: + The option's StdParam key. + kwargs: + The option properties. + """ + option_name = std_param.name.replace('_', '-') + if kwargs.get('type') == bool: + param_decls = [f'--{option_name}/--no-{option_name}'] + else: + param_decls = [f'--{option_name}'] + if kwargs.get('hide_input', False): + make_option_secret(kwargs, prompt=std_param.name.replace('_', ' ')) + return click.Option(param_decls, **kwargs) + + +@no_type_check +def select_std_options(tags: StdTags | list[StdTags] | str, + exclude: StdParams | list[StdParams] | None = None, + override: dict[StdParams, dict[str, Any]] | None = None, + formatters: dict[StdParams, ParameterFormatters] | None = None + ) -> list[click.Option]: + """ + Selects all or a subset of the defined standard Click options. + + Parameters: + tags: + A flag or a list of flags that define the option selection criteria. Each flag + is a combination of the StdTags. An option gets selected if it's StdParams.tags + property includes any of the provided flags. + If the tags is the string "all" all the standard options will be selected. + exclude: + An option or a list of options that should not to be included in the output even + though they match the tags criteria. + override: + A dictionary of standard options with overridden properties + formatters: + """ + if not isinstance(tags, list) and not isinstance(tags, str): + tags = [tags] + if exclude is None: + exclude = [] + elif not isinstance(exclude, list): + exclude = [exclude] + override = override or {} + formatters = formatters or {} + + def options_filter(std_param: StdParams) -> bool: + return any(tag in std_param.tags for tag in tags) and std_param not in exclude + + def option_params(std_param: StdParams) -> dict[str, Any]: + return override[std_param] if std_param in override else _std_options[std_param] + + if tags == 'all': + filtered_params = _std_options + else: + filtered_params = filter(options_filter, _std_options) + return [create_std_option(std_param, **option_params(std_param), + callback=formatters.get(std_param)) + for std_param in filtered_params] diff --git a/test/unit/cli/test_std_options.py b/test/unit/cli/test_std_options.py new file mode 100644 index 0000000..4d7aefa --- /dev/null +++ b/test/unit/cli/test_std_options.py @@ -0,0 +1,122 @@ +import os +import click +from click.testing import CliRunner +from exasol.python_extension_common.cli.std_options import ( + ParameterFormatters, + SECRET_DISPLAY, + StdTags, + StdParams, + create_std_option, + select_std_options +) + + +def test_parameter_formatters_1param(): + container_url_param = 'container_url' + cmd = click.Command('a_command') + ctx = click.Context(cmd) + opt = click.Option(['--version']) + formatters = ParameterFormatters() + formatters.set_formatter(container_url_param, 'http://my_server/{version}/my_stuff') + formatters(ctx, opt, '1.3.2') + assert ctx.params[container_url_param] == 'http://my_server/1.3.2/my_stuff' + + +def test_parameter_formatters_2params(): + container_url_param = 'container_url' + container_name_param = 'container_name' + cmd = click.Command('a_command') + ctx = click.Context(cmd) + opt1 = click.Option(['--version']) + opt2 = click.Option(['--user']) + formatters = ParameterFormatters() + formatters.set_formatter(container_url_param, 'http://my_server/{version}/{user}/my_stuff') + formatters.set_formatter(container_name_param, 'downloaded-{version}') + formatters(ctx, opt1, '1.3.2') + formatters(ctx, opt2, 'cezar') + assert ctx.params[container_url_param] == 'http://my_server/1.3.2/cezar/my_stuff' + assert ctx.params[container_name_param] == 'downloaded-1.3.2' + + +def test_create_std_option(): + opt = create_std_option(StdParams.bucketfs_name, type=str) + assert opt.name == StdParams.bucketfs_name.name + + +def test_create_std_option_bool(): + opt = create_std_option(StdParams.allow_override, type=bool) + assert opt.name == StdParams.allow_override.name + assert '--no-allow-override' in opt.secondary_opts + + +def test_create_std_option_secret(): + opt = create_std_option(StdParams.db_password, type=str, hide_input=True) + assert opt.hide_input + assert not opt.prompt_required + assert opt.default == SECRET_DISPLAY + + +def test_select_std_options(): + for tag in StdTags: + opts = {opt.name for opt in select_std_options(tag)} + expected_opts = {std_param.name for std_param in StdParams if tag in std_param.tags} + assert opts == expected_opts + + +def test_select_std_options_all(): + opts = {opt.name for opt in select_std_options('all')} + expected_opts = {std_param.name for std_param in StdParams} + assert opts == expected_opts + + +def test_select_std_options_restricted(): + opts = {opt.name for opt in select_std_options(StdTags.BFS)} + opts_onprem = {opt.name for opt in select_std_options(StdTags.BFS | StdTags.ONPREM)} + assert opts_onprem + assert len(opts) > len(opts_onprem) + assert opts.intersection(opts_onprem) == opts_onprem + + +def test_select_std_options_multi_tags(): + opts = {opt.name for opt in select_std_options([StdTags.BFS, StdTags.SLC])} + expected_opts_bfs = {std_param.name for std_param in StdParams + if StdTags.BFS in std_param.tags} + expected_opts_slc = {std_param.name for std_param in StdParams + if StdTags.SLC in std_param.tags} + expected_opts = expected_opts_bfs.union(expected_opts_slc) + assert opts == expected_opts + + +def test_select_std_options_with_exclude(): + opts = [opt.name for opt in select_std_options(StdTags.SLC, + exclude=StdParams.language_alias)] + assert StdParams.language_alias.name not in opts + + +def test_select_std_options_with_override(): + opts = {opt.name: opt for opt in select_std_options( + StdTags.SLC, override={StdParams.alter_system: {'type': bool, 'default': False}})} + assert not opts[StdParams.alter_system.name].default + + +def test_hidden_opt_with_envar(): + """ + This test checks the mechanism of providing a value of a confidential parameter + via an environment variable. + """ + std_param = StdParams.db_password + envar_name = std_param.name.upper() + param_value = 'my_password' + + def func(**kwargs): + assert std_param.name in kwargs + assert kwargs[std_param.name] == param_value + + opt = create_std_option(std_param, type=str, hide_input=True) + cmd = click.Command('do_something', params=[opt], callback=func) + runner = CliRunner() + os.environ[envar_name] = param_value + try: + runner.invoke(cmd) + finally: + os.environ.pop(envar_name) From e3177cb610deb3c4865ddf7df71dd862f7e59f69 Mon Sep 17 00:00:00 2001 From: Mikhail Beck Date: Tue, 1 Oct 2024 15:10:47 +0100 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> --- doc/changes/unreleased.md | 2 +- exasol/python_extension_common/cli/std_options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 51beab6..c0d22a5 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,4 +2,4 @@ ## Features -* #66: Implement a standard CLI command builder. +* #66: Implement a standard CLI options builder. diff --git a/exasol/python_extension_common/cli/std_options.py b/exasol/python_extension_common/cli/std_options.py index 741b797..556944d 100644 --- a/exasol/python_extension_common/cli/std_options.py +++ b/exasol/python_extension_common/cli/std_options.py @@ -24,7 +24,7 @@ class ParameterFormatters: the update if the value of the parameter dressed with the callback is None. """ def __init__(self): - self._formatters = {} + self._formatters : Dict[str, str] = {} def __call__(self, ctx: click.Context, param: click.Parameter, value: Any | None) -> Any | None: From bcf3543481da4738fe57a1c552998f1a2d6aac4a Mon Sep 17 00:00:00 2001 From: mibe Date: Tue, 1 Oct 2024 15:19:13 +0100 Subject: [PATCH 4/4] #66 Addressed the review comments --- exasol/python_extension_common/cli/std_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exasol/python_extension_common/cli/std_options.py b/exasol/python_extension_common/cli/std_options.py index 556944d..1bafbbd 100644 --- a/exasol/python_extension_common/cli/std_options.py +++ b/exasol/python_extension_common/cli/std_options.py @@ -24,7 +24,7 @@ class ParameterFormatters: the update if the value of the parameter dressed with the callback is None. """ def __init__(self): - self._formatters : Dict[str, str] = {} + self._formatters: dict[str, str] = {} def __call__(self, ctx: click.Context, param: click.Parameter, value: Any | None) -> Any | None: