From df7a2726a43709ab45037dbf86db4a70aaaab7d9 Mon Sep 17 00:00:00 2001
From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com>
Date: Wed, 18 Sep 2024 15:51:54 +0200
Subject: [PATCH] #414: Ignore rsyslogd related errors in db (#415)
fixes #414
---
doc/changes/changes_3.2.0.md | 6 +-
.../db_container_log_thread.py | 30 +++++-
noxfile.py | 3 +-
.../test_db_container_log_thread.py | 99 +++++++++++++++++++
4 files changed, 130 insertions(+), 8 deletions(-)
create mode 100644 test/integration/test_db_container_log_thread.py
diff --git a/doc/changes/changes_3.2.0.md b/doc/changes/changes_3.2.0.md
index a279d9475..f753fb957 100644
--- a/doc/changes/changes_3.2.0.md
+++ b/doc/changes/changes_3.2.0.md
@@ -1,8 +1,8 @@
-# Integration-Test-Docker-Environment 3.2.0, released t.b.d.
+# Integration-Test-Docker-Environment 3.2.0, released 2024-09-18
## Summary
-Updated dependency constraints and supported Exasol versions.
+Updated dependency constraints and supported Exasol versions. Also, ignore crashes of rsyslogd in the docker-db.
### Supported Exasol Versions
@@ -12,4 +12,6 @@ Updated dependency constraints and supported Exasol versions.
## Dependencies
## Changes
+
#412: Add latest Docker-DB versions
+#414: Ignore rsyslogd related errors in db
\ No newline at end of file
diff --git a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py
index 07cf7ce60..cdf4b6a2d 100644
--- a/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py
+++ b/exasol_integration_test_docker_environment/lib/test_environment/database_waiters/db_container_log_thread.py
@@ -2,6 +2,7 @@
import time
from pathlib import Path
from threading import Thread
+from typing import Callable, Optional
from docker.models.containers import Container
@@ -21,6 +22,28 @@ def __init__(self, container: Container, logger, log_file: Path, description: st
self.previous_timestamp = None
self.current_timestamp = None
self.error_message = None
+ self.ignore_error_return_codes = (
+ "(membership) returned with state 1", # exclude webui not found in 7.0.0
+ "rsyslogd) returned with state 1" # exclude rsyslogd which might crash when running itde under lima
+ )
+
+ def _contains_error(self, log_line: str) -> bool:
+ def ignore_sshd(log_line_local):
+ return "sshd was not started" in log_line_local
+
+ def ignore_return_code(log_line_local):
+ return any(x in log_line_local for x in self.ignore_error_return_codes)
+
+ def contains(substr: str, ignore: Optional[Callable[[str], bool]] = None):
+ if not substr in log_line:
+ return False
+ return ignore is None or not ignore(log_line)
+
+ return (
+ contains("error", ignore_sshd)
+ or contains("exception")
+ or contains("returned with state 1", ignore_return_code)
+ )
def stop(self):
self.logger.info("Stop ContainerLogThread")
@@ -38,10 +61,7 @@ def run(self):
still_running_logger.log()
log_handler.handle_log_lines(log)
log_line = log.decode("utf-8").lower()
- if ("error" in log_line and not "sshd was not started" in log_line) \
- or "exception" in log_line \
- or ("returned with state 1" in log_line
- and not "(membership) returned with state 1" in log_line): # exclude webui not found in 7.0.0
+ if self._contains_error(log_line):
self.logger.info("ContainerLogHandler error message, %s", log_line)
self.error_message = log_line
self.finish = True
@@ -50,4 +70,4 @@ def run(self):
time.sleep(1)
except Exception as e:
self.finish = True
- self.logger.exception("Caught exception in DBContainerLogThread.run.")
\ No newline at end of file
+ self.logger.exception("Caught exception in DBContainerLogThread.run.")
diff --git a/noxfile.py b/noxfile.py
index a8f0f77b5..87197fb19 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -201,7 +201,8 @@ def run_minimal_tests(session: nox.Session, db_version: str):
"test_termination_handler.py",
],
"new-itest": [
- "test_cli_environment.py"
+ "test_cli_environment.py",
+ "test_db_container_log_thread.py"
],
"unit": "./test/unit",
}
diff --git a/test/integration/test_db_container_log_thread.py b/test/integration/test_db_container_log_thread.py
new file mode 100644
index 000000000..32093f75c
--- /dev/null
+++ b/test/integration/test_db_container_log_thread.py
@@ -0,0 +1,99 @@
+import logging
+import tempfile
+import time
+from pathlib import Path
+from typing import List
+
+import pytest
+
+from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient
+from exasol_integration_test_docker_environment.lib.test_environment.database_waiters.db_container_log_thread import \
+ DBContainerLogThread
+
+def _build_docker_command(logs: List[str]):
+ """
+ Builds a bash while loop command which can be used to print logs.
+ Args:
+ logs: List of logs to print, each in one line.
+
+ Returns:
+ Something like ["bash", "-c", "while true; do echo 'Test'; done"]
+ """
+ echo_commands = [f"echo '{log}'" for log in logs]
+ echo_commdands_str = ";".join(echo_commands)
+ bash_command = f"while true; do {echo_commdands_str}; sleep 0.1; done"
+ return ["bash", "-c", bash_command]
+
+def _run_container_log_thread(logger, logs: List[str]) -> str:
+ """
+ Starts a dummy docker container which prints logs in an endless loop, and calls DBContainerLogThread on that container.
+
+ Returns: resulting DBContainerLogThread.error_message
+ """
+ with tempfile.TemporaryDirectory() as tmpDir:
+ with ContextDockerClient(timeout=3600) as client:
+ try:
+ container = client.containers.run("ubuntu", _build_docker_command(logs), detach=True)
+ thread = DBContainerLogThread(container, logger, Path(tmpDir) / "log.txt", "test")
+ thread.start()
+ time.sleep(2)
+ thread.stop()
+ finally:
+ container.stop()
+ container.remove()
+ return thread.error_message
+
+
+@pytest.fixture
+def test_logger():
+ return logging.Logger(__name__)
+
+def test_container_log_thread_no_error(test_logger) -> None:
+ """
+ Integration test which verifies that the DBContainerLogThread returns no error message if no error is logged.
+ """
+ error_message = _run_container_log_thread(test_logger, ["test", "something", "db started"])
+ assert error_message is None
+
+def test_container_log_thread_error(test_logger) -> None:
+ """
+ Integration test which verifies that the DBContainerLogThread returns error message if error is logged.
+ """
+ error_message = _run_container_log_thread(test_logger, ["confd returned with state 1"])
+ assert "confd returned with state 1\n" in error_message
+
+def test_container_log_thread_ignore_rsyslogd(test_logger) -> None:
+ """
+ Integration test which verifies that the DBContainerLogThread returns no error message if rsyslogd crashes.
+ """
+ rsys_logd_logs = [
+ "[2024-09-17 14:12:20.335085 +00:00] child 58687 (Part:9 Node:0 rsyslogd) returned with state 1.",
+ "[2024-09-17 14:12:20.336886 +00:00] Started /sbin/rsyslogd with PID:58688 UID:0 GID:0 Part:9 Node:0",
+ "[2024-09-17 14:12:20.336967 +00:00] 30 auto-restarted processes exited in the last 0 seconds. Starting to delay process death handling."
+ ]
+ error_message = _run_container_log_thread(test_logger, rsys_logd_logs)
+ assert error_message is None
+
+def test_container_log_thread_ignore_sshd(test_logger) -> None:
+ """
+ Integration test which verifies that the DBContainerLogThread returns no error message if sshd crashes.
+ """
+ sshd_logs = [
+ "[2024-09-17 14:12:20.335085 +00:00] error : sshd was not started.",
+ ]
+ error_message = _run_container_log_thread(test_logger, sshd_logs)
+ assert error_message is None
+
+def test_container_log_thread_exception(test_logger) -> None:
+ """
+ Integration test which verifies that the DBContainerLogThread returns an error message if an exception was thrown.
+ """
+ sshd_logs = [
+ "Traceback (most recent call last):",
+ 'File "/opt/cos/something.py", line 364, in runcode',
+ ' coro = func()',
+ ' File "", line 1, in ',
+ 'Exception: bad thing happend'
+ ]
+ error_message = _run_container_log_thread(test_logger, sshd_logs)
+ assert "exception: bad thing happend\n" in error_message