diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d634da3..beb855b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - The lifetime of auto generated TLS certificates is now configurable with the role and roleGroup config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#598]). +- Run a `containerdebug` process in the background of each HBase container to collect debugging information ([#605]). ### Fixed @@ -15,6 +16,7 @@ [#594]: https://github.com/stackabletech/hbase-operator/pull/594 [#598]: https://github.com/stackabletech/hbase-operator/pull/598 +[#605]: https://github.com/stackabletech/hbase-operator/pull/605 ## [24.11.0] - 2024-11-18 diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 99077dcc..97f022fc 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -881,6 +881,7 @@ fn build_rolegroup_statefulset( {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & bin/hbase {hbase_role_name_in_command} start & wait_for_termination $! {create_vector_shutdown_file_command} @@ -893,6 +894,11 @@ fn build_rolegroup_statefulset( create_vector_shutdown_file_command(STACKABLE_LOG_DIR), }]) .add_env_vars(merged_env) + // Needed for the `containerdebug` process to log it's tracing information to. + .add_env_var( + "CONTAINERDEBUG_LOG_DIRECTORY", + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ) .add_volume_mount("hbase-config", HBASE_CONFIG_TMP_DIR) .context(AddVolumeMountSnafu)? .add_volume_mount("hdfs-discovery", HDFS_DISCOVERY_TMP_DIR) diff --git a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 index f862651c..31a27a68 100644 --- a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 @@ -39,6 +39,9 @@ commands: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + resources: + memory: + limit: 1536Mi roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 index ac1bed8c..53e9a98e 100644 --- a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 @@ -45,6 +45,9 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + resources: + memory: + limit: 1Gi roleGroups: default: configOverrides: diff --git a/tests/templates/kuttl/smoke/31-assert.yaml b/tests/templates/kuttl/smoke/31-assert.yaml new file mode 100644 index 00000000..25f259d3 --- /dev/null +++ b/tests/templates/kuttl/smoke/31-assert.yaml @@ -0,0 +1,9 @@ +--- +# This test checks if the containerdebug-state.json file is present and valid +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-master-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"' + - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-regionserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"' + - script: kubectl exec -n $NAMESPACE --container hbase test-hbase-restserver-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valif JSON"'