#34: Added validation for results of tests and security_scan

exasol_script_languages_container_ci/lib/ci_security_scan.py

@@ -1,25 +1,41 @@
 import logging
 from pathlib import Path
-from typing import Tuple
+from typing import Tuple, Callable
 
 from exasol_script_languages_container_tool.lib.api import security_scan
-from exasol_script_languages_container_tool.lib.tasks.security_scan.security_scan import AllScanResult
 
-import exasol_script_languages_container_ci.lib.common
+from exasol_script_languages_container_ci.lib.common import print_file, print_docker_images
+
+
+def _print_docker_images_function():
+    print_docker_images(logging.info)
+
+
+def _print_file_function(path: Path):
+    print_file(path, logging.info)
 
 
 class CISecurityScan:
 
-    def run_security_scan(self, flavor_path: Tuple[str, ...]) -> AllScanResult:
+    def __init__(self,
+                 print_file_function: Callable[[Path], None] = _print_file_function,
+                 print_docker_images_function: Callable[[], None] = _print_docker_images_function
+                 ):
+        self._print_docker_images_function = print_docker_images_function
+        self._print_file_function = print_file_function
+
+    def run_security_scan(self,
+                          flavor_path: Tuple[str, ...]):
         """
         Run security scan and print result
         """
 
         logging.info(f"Running command 'security_scan' with parameters {locals()}")
         security_scan_result = security_scan(flavor_path=flavor_path, workers=7)
-        exasol_script_languages_container_ci.lib.common.print_docker_images(logging.info)
         logging.info("============= SECURITY REPORT ===========")
-        # Important: Call print_file over the global module name, otherwise the patch in the unit-test does not work!
-        exasol_script_languages_container_ci.lib.common.print_file(
-            Path() / ".build_output" / "security_scan" / "security_report", logging.info)
-        return security_scan_result
+        print(security_scan_result)
+        #if security_scan_result.report_path.exists():
+        self._print_file_function(Path(security_scan_result.report_path))
+        self._print_docker_images_function()
+        if not security_scan_result.scans_are_ok:
+            raise AssertionError("Some security scans not successful.")

exasol_script_languages_container_ci/lib/ci_test.py

@@ -1,38 +1,94 @@
 import logging
-from typing import Tuple
+from pathlib import Path
+from typing import Tuple, Callable, Protocol, Optional
 
-import click
+from exasol_integration_test_docker_environment.cli.options.test_environment_options import LATEST_DB_VERSION
 from exasol_script_languages_container_tool.lib.api.run_db_tests import run_db_test
 
-from exasol_script_languages_container_ci.lib.common import print_docker_images
+from exasol_script_languages_container_ci.lib.common import print_docker_images, print_file
 
 from exasol_script_languages_container_tool.lib.tasks.test.test_container import AllTestsResult
 
 
+def _print_docker_images_function():
+    print_docker_images(logging.info)
+
+
+def _print_file_function(path: Path):
+    print_file(path, logging.info)
+
+
+class RunDBTestsProtocol(Protocol):
+    def __call__(self,
+                 flavor_path: Tuple[str, ...],
+                 release_goal: Tuple[str, ...],
+                 test_folder: Tuple[str, ...],
+                 test_container_folder: str,
+                 workers: int,
+                 source_docker_username: str,
+                 source_docker_password: str) -> AllTestsResult:
+        ...
+
+
 class CIExecuteTest:
 
+    def __init__(self,
+                 print_file_function: Callable[[Path], None] = _print_file_function,
+                 print_docker_images_function: Callable[[], None] = _print_docker_images_function,
+                 run_db_test: RunDBTestsProtocol = run_db_test):
+        self._run_db_test = run_db_test
+        self._print_docker_images_function = print_docker_images_function
+        self._print_file_function = print_file_function
+
     def execute_tests(self,
                       flavor_path: Tuple[str, ...],
                       docker_user: str,
                       docker_password: str,
-                      test_container_folder: str) -> Tuple[AllTestsResult, AllTestsResult]:
+                      test_container_folder: str):
         """
         Run db tests
         """
+        db_tests_are_ok = self.run_db_tests(flavor_path=flavor_path,
+                                            docker_user=docker_user,
+                                            docker_password=docker_password,
+                                            test_container_folder=test_container_folder)
+        linker_namespace_tests_are_ok = self.run_linker_namespace_tests(flavor_path=flavor_path,
+                                                                        docker_user=docker_user,
+                                                                        docker_password=docker_password,
+                                                                        test_container_folder=test_container_folder)
+        self._print_docker_images_function()
+        tests_are_ok = db_tests_are_ok and linker_namespace_tests_are_ok
+        if not tests_are_ok:
+            raise AssertionError("Not all tests are ok!")
+
+    def run_db_tests(self, flavor_path: Tuple[str, ...],
+                     docker_user: str,
+                     docker_password: str,
+                     test_container_folder: str) -> bool:
         logging.info(f"Running command 'run_db_test' for flavor-path {flavor_path}")
-        run_db_test_flavor_result = \
-            run_db_test(flavor_path=flavor_path,
-                        workers=7,
-                        source_docker_username=docker_user,
-                        source_docker_password=docker_password,
-                        test_container_folder=test_container_folder)
+        db_test_result = \
+            self._run_db_test(flavor_path=flavor_path,
+                              test_folder=tuple(),
+                              release_goal=('release',),
+                              workers=7,
+                              source_docker_username=docker_user,
+                              source_docker_password=docker_password,
+                              test_container_folder=test_container_folder)
+        self._print_file_function(db_test_result.command_line_output_path)
+        return db_test_result.tests_are_ok
+
+    def run_linker_namespace_tests(self,
+                                   flavor_path: Tuple[str, ...],
+                                   docker_user: str,
+                                   docker_password: str,
+                                   test_container_folder: str) -> bool:
         logging.info(f"Running command 'run_db_test' for linker_namespace_sanity for flavor-path {flavor_path}")
-        run_db_test_linkernamespace = \
-            run_db_test(flavor_path=flavor_path, workers=7,
-                        test_folder=("test/linker_namespace_sanity",),
-                        release_goal=("base_test_build_run",),
-                        source_docker_username=docker_user,
-                        source_docker_password=docker_password,
-                        test_container_folder=test_container_folder)
-        print_docker_images(logging.info)
-        return run_db_test_flavor_result, run_db_test_linkernamespace
+        linker_namespace_test_result = \
+            self._run_db_test(flavor_path=flavor_path, workers=7,
+                              test_folder=("test/linker_namespace_sanity",),
+                              release_goal=("base_test_build_run",),
+                              source_docker_username=docker_user,
+                              source_docker_password=docker_password,
+                              test_container_folder=test_container_folder)
+        self._print_file_function(linker_namespace_test_result.command_line_output_path)
+        return linker_namespace_test_result.tests_are_ok

test/conftest.py

@@ -1,13 +1,28 @@
 import json
 import os
-from dataclasses import dataclass
-
+from pathlib import Path
 from tempfile import TemporaryDirectory
-from unittest.mock import Mock, patch, MagicMock
+from unittest.mock import patch, MagicMock
 
-import click
 import pytest
 
+script_path = Path(__file__).absolute().parent
+
+
+@pytest.fixture
+def resources_path() -> Path:
+    return script_path / "integration_tests/resources"
+
+
+@pytest.fixture
+def flavors_path(resources_path: Path) -> Path:
+    return resources_path / "flavors"
+
+
+@pytest.fixture
+def test_container_folder(resources_path: Path) -> Path:
+    return resources_path / "test_container"
+
 
 @pytest.fixture(autouse=True)
 def tmp_test_dir():
@@ -21,6 +36,7 @@ def tmp_test_dir():
         yield temp_dir
         os.chdir(old_dir)
 
+
 @pytest.fixture
 def config_file(tmp_path_factory):
     config_file_path = tmp_path_factory.mktemp("config") / "build_config.json"
@@ -50,4 +66,4 @@ def git_access_mock():
     git_access_mock.get_head_commit_sha_of_branch.return_value = "123"
     git_access_mock.get_files_of_commit.return_value = ["src/udfclient.cpp"]
     git_access_mock.get_last_commit_message.return_value = "last commit"
-    return git_access_mock
+    return git_access_mock

test/integration_tests/resources/flavors/broken-security-scan/flavor_base/security_scan/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+echo Running scan...
+mkdir -p $1
+echo Report 123 >> "$1/report.txt"
+exit 1

test/integration_tests/resources/flavors/broken-security-scan/flavor_base/testconfig

@@ -0,0 +1,3 @@
+generic_language_tests=
+test_folders=functioning_test
+# trailing new line required!

test/integration_tests/resources/flavors/broken-test/flavor_base/base_test_build_run/Dockerfile

@@ -0,0 +1 @@
+FROM ubuntu:22.04

test/integration_tests/resources/flavors/broken-test/flavor_base/build_steps.py

@@ -0,0 +1,52 @@
+from typing import Dict
+
+from exasol_script_languages_container_tool.lib.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask
+
+
+class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "base_test_build_run"
+
+    def requires_tasks(self):
+        return {}
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "flavor_test_build_run"
+
+    def requires_tasks(self):
+        return {"base_test_build_run": AnalyzeBaseTestBuildRun}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "release"
+
+    def requires_tasks(self):
+        return {}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class SecurityScan(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "security_scan"
+
+    def requires_tasks(self):
+        return {"release": AnalyzeRelease}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"

test/integration_tests/resources/flavors/broken-test/flavor_base/flavor_test_build_run/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{base_test_build_run}}
+
+RUN mkdir /conf /buckets
+
+
+RUN mkdir /exaudf

test/integration_tests/resources/flavors/broken-test/flavor_base/language_definition

@@ -0,0 +1 @@
+PYTHON3_TEST=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3

test/integration_tests/resources/flavors/broken-test/flavor_base/release/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:22.04
+RUN mkdir /conf /buckets
+RUN mkdir /exaudf

test/integration_tests/resources/flavors/broken-test/flavor_base/security_scan/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{release}}
+
+RUN echo "Building security scan..."
+
+COPY security_scan/test.sh /test.sh
+ENTRYPOINT ["/test.sh"]

test/integration_tests/resources/flavors/real-test-flavor/flavor_base/testconfig → test/integration_tests/resources/flavors/broken-test/flavor_base/testconfig

@@ -1,3 +1,3 @@
 generic_language_tests=
-test_folders=empty_test
+test_folders=broken_test
 # trailing new line required!

test/integration_tests/resources/flavors/broken-test/flavor_customization/Dockerfile

@@ -0,0 +1 @@
+FROM {{flavor_base_deps}}

test/integration_tests/resources/flavors/functioning/flavor_base/base_test_build_run/Dockerfile

@@ -0,0 +1 @@
+FROM ubuntu:22.04

test/integration_tests/resources/flavors/functioning/flavor_base/build_steps.py

@@ -0,0 +1,52 @@
+from typing import Dict
+
+from exasol_script_languages_container_tool.lib.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask
+
+
+class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "base_test_build_run"
+
+    def requires_tasks(self):
+        return {}
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask):
+
+    def get_build_step(self) -> str:
+        return "flavor_test_build_run"
+
+    def requires_tasks(self):
+        return {"base_test_build_run": AnalyzeBaseTestBuildRun}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "release"
+
+    def requires_tasks(self):
+        return {}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class SecurityScan(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "security_scan"
+
+    def requires_tasks(self):
+        return {"release": AnalyzeRelease}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"

test/integration_tests/resources/flavors/functioning/flavor_base/flavor_test_build_run/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{base_test_build_run}}
+
+RUN mkdir /conf /buckets
+
+
+RUN mkdir /exaudf

test/integration_tests/resources/flavors/functioning/flavor_base/language_definition

@@ -0,0 +1 @@
+PYTHON3_TEST=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3

test/integration_tests/resources/flavors/functioning/flavor_base/release/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:22.04
+RUN mkdir /conf /buckets
+RUN mkdir /exaudf

test/integration_tests/resources/flavors/functioning/flavor_base/security_scan/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{release}}
+
+RUN echo "Building security scan..."
+
+COPY security_scan/test.sh /test.sh
+ENTRYPOINT ["/test.sh"]

test/integration_tests/resources/flavors/functioning/flavor_base/security_scan/test.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo Running scan...
+mkdir -p $1
+echo Report 123 >> "$1/report.txt"