Merge pull request #439 from boozallen/fixdocker-compose

[#4] switch from docker compose to testcontainers

README.md

@@ -3,7 +3,9 @@
 [![Maven Central](https://img.shields.io/maven-central/v/com.boozallen.aissemble/aissemble-root.svg)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22com.boozallen.aissemble%22%20AND%20a%3A%22aissemble-root%22)
 ![PyPI](https://img.shields.io/pypi/v/aissemble-foundation-core-python?logo=python&logoColor=gold)
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/aissemble-foundation-core-python?logo=python&logoColor=gold)
-[![Publish to GitHub Pages](https://github.com/boozallen/aissemble/actions/workflows/publish.yml/badge.svg)](https://github.com/boozallen/aissemble/actions/workflows/publish.yml)
+
+[![Build](https://github.com/boozallen/aissemble/actions/workflows/build.yml/badge.svg)](https://github.com/boozallen/aissemble/actions/workflows/build.yml)
+[![Publish Docs](https://github.com/boozallen/aissemble/actions/workflows/publish.yml/badge.svg)](https://github.com/boozallen/aissemble/actions/workflows/publish.yml)
 ## aiSSEMBLE Overview
 
 ### Purpose of the aiSSEMBLE

extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/src/aissemble_encrypt/vault_config.py

@@ -64,18 +64,23 @@ def secrets_host_url(self):
         Returns the secrets host url.
         This should only be used for integration tests, and not otherwise.
         """
-        return self.properties["secrets.host.url"]
+        if "SECRETS_HOST_URL" in os.environ:
+            # host and port of the aissemble-vault container
+            return os.environ["SECRETS_HOST_URL"]
+        else:
+            return self.properties["secrets.host.url"]
 
     @staticmethod
-    def validate_container_start():
+    def validate_container_start(port):
         started = False
         max_wait = 20
         wait = 0
         while wait < max_wait:
-            logger.info("Waiting for Vault to start")
+            url = f"http://localhost:{port}/v1/sys/health"
+            logger.info(f"Waiting for Vault to start at {url}")
 
             try:
-                requests.get("http://localhost:8200/v1/sys/health")
+                requests.get(url)
                 logger.info("Vault started successfully!")
                 started = True
                 break

extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/environment.py

@@ -8,8 +8,7 @@
 # #L%
 ###
 from krausening.logging import LogManager
-from container.safe_docker_container import SafeDockerContainer
-from testcontainers.core.waiting_utils import wait_for
+from testcontainers.core.container import DockerContainer
 from aissemble_encrypt.vault_config import VaultConfig
 from importlib import metadata
 import packaging.version
@@ -44,12 +43,14 @@ def select_krausening_extensions():
         os.environ["KRAUSENING_EXTENSIONS"] = "tests/resources/krausening/arm64"
 
 
-def start_container(context, docker_image, feature):
+def start_container(context, docker_image, feature) -> int:
     logger.info(f"Starting container: {docker_image}")
-    context.test_container.with_bind_ports(8200, 8200)
+    context.test_container.with_exposed_ports(8200)
     context.test_container.start()
-    wait_for(VaultConfig.validate_container_start)
-
+    extport = context.test_container.get_exposed_port(8200)
+    if not VaultConfig.validate_container_start(extport):
+        raise Exception("Vault failed to start")
+    return extport
 
 def before_feature(context, feature):
     if "integration" in feature.tags:
@@ -63,8 +64,9 @@ def before_feature(context, feature):
         version = metadata.version("aissemble-extensions-encryption-vault-python")
         docker_image += version_to_tag(version)
 
-        context.test_container = SafeDockerContainer(docker_image)
-        start_container(context, docker_image, feature)
+        context.test_container = DockerContainer(docker_image)
+        port = start_container(context, docker_image, feature)
+        os.environ["SECRETS_HOST_URL"] = f"http://127.0.0.1:{port}"
 
         root_key_tuple = context.test_container.exec("cat /root_key.txt")
         secrets_root_key = root_key_tuple.output.decode()

foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/testcontainers_kafka_kraft.py

@@ -1,11 +1,11 @@
 from kafka import KafkaConsumer
 from kafka.errors import KafkaError, UnrecognizedBrokerVersion, NoBrokersAvailable
 
-from container.safe_docker_container import SafeDockerContainer
+from testcontainers.core.container import DockerContainer
 from testcontainers.core.waiting_utils import wait_container_is_ready
 
 
-class KafkaKraftContainer(SafeDockerContainer):
+class KafkaKraftContainer(DockerContainer):
     """
     Kafka container.
 

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml

@@ -189,28 +189,6 @@
             <id>integration-test</id>
             <build>
                 <plugins>
-                    <plugin>
-                        <groupId>org.codehaus.mojo</groupId>
-                        <artifactId>exec-maven-plugin</artifactId>
-                        <version>${version.exec.maven.plugin}</version>
-                        <executions>
-                            <execution>
-                                <id>ensure-docker-compose-installed</id>
-                                <phase>initialize</phase>
-                                <goals>
-                                    <goal>exec</goal>
-                                </goals>
-                                <configuration>
-                                    <executable>sh</executable>
-                                    <arguments>
-                                        <argument>
-                                            ${project.basedir}/tests/resources/integration-test-resources/docker-compose-setup.sh
-                                        </argument>
-                                    </arguments>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
                     <plugin>
                         <artifactId>maven-resources-plugin</artifactId>
                         <executions>

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/environment.py

@@ -8,12 +8,15 @@
 # #L%
 ###
 import os
-import platform
 import json
+import time
+import packaging.version
+from importlib import metadata
 from pyspark.sql import SparkSession
 from krausening.logging import LogManager
-from testcontainers.compose import DockerCompose
+from testcontainers.core.container import DockerContainer
 from aissemble_test_data_delivery_pyspark_model.generated import environment_base
+from aissemble_encrypt.vault_config import VaultConfig
 
 """
 Behave test environment setup to configure Spark for unit tests.
@@ -33,15 +36,6 @@ def before_all(context):
 
     print("Created spark session for tests...")
 
-    os.environ["S3Test_FS_PROVIDER"] = "s3"
-    os.environ["S3Test_FS_ACCESS_KEY_ID"] = "000000000000"
-    os.environ["S3Test_FS_SECRET_ACCESS_KEY"] = (
-        "E3FF2839C048B25C084DEBE9B26995E310250568"
-    )
-    os.environ["S3Test_FS_SECURE"] = "False"
-    os.environ["S3Test_FS_HOST"] = "localhost"
-    os.environ["S3Test_FS_PORT"] = "4566"
-
 
 def after_all(context):
     environment_base.cleanup()
@@ -63,47 +57,104 @@ def after_scenario(context, scenario):
 
 def before_feature(context, feature):
     if "integration" in feature.tags:
-        test_staging_path = "target/generated-sources/docker-compose-files"
-        logger.info(
-            f"Starting services defined in Docker Compose file at {test_staging_path}"
-        )
-
-        compose = DockerCompose(test_staging_path, "docker-compose.yml")
-        context.docker_compose_containers = compose
-
-        compose.start()
-        wait_for_docker_url = "http://localhost:4566"
-        logger.info(
-            f"Waiting for Docker Compose services to start - waiting for a response from {wait_for_docker_url}"
-        )
-        compose.wait_for(wait_for_docker_url)
+        logger.info("Starting Test container services")
+        context.test_containers = []
+        setup_vault(context)
+        setup_s3_local(context)
+
+
+def setup_vault(context):
+    docker_image = "ghcr.io/boozallen/aissemble-vault:"
+    # append current version to docker image
+    # pyproject.toml has a "version" property, e.g. version = "0.12.0.dev"
+    # using major, minor, patch and -SNAPSHOT if dev
+    version = metadata.version("aissemble-extensions-encryption-vault-python")
+    docker_image += version_to_tag(version)
+    vault = DockerContainer(docker_image)
+    port = start_container(vault, 8200, VaultConfig.validate_container_start)
+    context.test_containers.append(vault)
+    os.environ["SECRETS_HOST_URL"] = f"http://127.0.0.1:{port}"
+
+    root_key_tuple = vault.exec("cat /root_key.txt")
+    secrets_root_key = root_key_tuple.output.decode()
+    os.environ["SECRETS_ROOT_KEY"] = secrets_root_key
+
+    unseal_keys_tuple = vault.exec("cat /unseal_keys.txt")
+    unseal_keys_txt = unseal_keys_tuple.output.decode()
+    unseal_keys_json = json.loads(unseal_keys_txt)
+    secrets_unseal_keys = ",".join(unseal_keys_json)
+    os.environ["SECRETS_UNSEAL_KEYS"] = secrets_unseal_keys
+
+    transit_client_token_tuple = vault.exec("cat /transit_client_token.txt")
+    transit_client_token_txt = transit_client_token_tuple.output.decode()
+    transit_client_token_json = json.loads(transit_client_token_txt)
+    encrypt_client_token = transit_client_token_json["auth"]["client_token"]
+    os.environ["ENCRYPT_CLIENT_TOKEN"] = encrypt_client_token
+
+
+def setup_s3_local(context):
+    localstack = DockerContainer("localstack/localstack:latest")
+    localstack.with_env("SERVICES", "s3")
+    port = start_container(localstack, 4566, lambda _: test_aws(localstack))
+    context.test_containers.append(localstack)
+    os.environ["S3Test_FS_PROVIDER"] = "s3"
+    os.environ["S3Test_FS_ACCESS_KEY_ID"] = "000000000000"
+    os.environ["S3Test_FS_SECRET_ACCESS_KEY"] = (
+        "E3FF2839C048B25C084DEBE9B26995E310250568"
+    )
+    os.environ["S3Test_FS_SECURE"] = "False"
+    os.environ["S3Test_FS_HOST"] = "localhost"
+    os.environ["S3Test_FS_PORT"] = f"{port}"
 
-        root_key_tuple = context.docker_compose_containers.exec_in_container(
-            ["cat", "/root_key.txt"], "vault"
-        )
-        secrets_root_key = root_key_tuple[0]
-        os.environ["SECRETS_ROOT_KEY"] = secrets_root_key
 
-        unseal_keys_tuple = context.docker_compose_containers.exec_in_container(
-            ["cat", "/unseal_keys.txt"], "vault"
-        )
-        unseal_keys_txt = unseal_keys_tuple[0]
-        unseal_keys_json = json.loads(unseal_keys_txt)
-        secrets_unseal_keys = ",".join(unseal_keys_json)
-        os.environ["SECRETS_UNSEAL_KEYS"] = secrets_unseal_keys
-
-        transit_client_token_tuple = (
-            context.docker_compose_containers.exec_in_container(
-                ["cat", "/transit_client_token.txt"], "vault"
-            )
+def after_feature(context, feature):
+    if hasattr(context, "test_containers"):
+        logger.info("Stopping Test container services")
+        for container in context.test_containers:
+            logger.info(f"...stopping {container.image}")
+            container.stop()
+
+
+def start_container(container, port, healthcheck=lambda x: True) -> int:
+    logger.info(f"Starting container: {container.image}")
+    container.with_exposed_ports(port)
+    container.start()
+    extport = container.get_exposed_port(port)
+    if not healthcheck(extport):
+        raise Exception(f"Failed to start {container.image}")
+    return extport
+
+
+def version_to_tag(version_str: str) -> str:
+    """Convert a python version into a docker tag for the same version.
+
+    Args:
+        version_str (str): The version string to convert.
+    Returns:
+        str: The docker tag for the version.
+    """
+    version = packaging.version.parse(version_str)
+    tag = version.base_version
+    if version.pre:
+        tag += "-" + "".join([str(x) for x in version.pre])
+    if version.is_devrelease:
+        tag += "-SNAPSHOT"
+    return tag
+
+
+def test_aws(container):
+    started = False
+    tries = 0
+    exitcode = -1
+    while exitcode != 0 and tries < 20:
+        logger.info("Waiting for s3 to start...")
+        tries += 1
+        exitcode, _ = container.exec(
+            "bash -c 'AWS_ACCESS_KEY_ID=fake AWS_SECRET_ACCESS_KEY=fake aws --endpoint-url=http://localhost:4566 s3 ls'"
         )
-        transit_client_token_txt = transit_client_token_tuple[0]
-        transit_client_token_json = json.loads(transit_client_token_txt)
-        encrypt_client_token = transit_client_token_json["auth"]["client_token"]
-        os.environ["ENCRYPT_CLIENT_TOKEN"] = encrypt_client_token
+        if exitcode == 0:
+            started = True
+        else:
+            time.sleep(1)
 
-
-def after_feature(context, feature):
-    if "integration" in feature.tags:
-        logger.info("Stopping Docker Compose services")
-        context.docker_compose_containers.stop()
+    return started