From ee2a7f5745091feec0ad7c0260b909e88714057c Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 20 Sep 2023 15:51:12 +0200 Subject: [PATCH] lib.sh: Split containerd restart from configure_cc_containerd Let's do this as that will make it easier for configuring / reconfiguring containerd, which will be necessary in the coming up commits / PRs. And here we're also adapting the previous tests to using this change. Signed-off-by: ChengyuZhu6 --- integration/confidential/lib.sh | 17 ++++++++------- .../containerd/confidential/tests_common.sh | 2 +- .../kubernetes/confidential/agent_image.bats | 4 +++- .../confidential/sealed_secret.bats | 9 ++------ integration/kubernetes/confidential/sev.bats | 3 ++- integration/kubernetes/confidential/snp.bats | 3 ++- .../kubernetes/confidential/tests_common.sh | 21 ++++++++++++++++++- 7 files changed, 40 insertions(+), 19 deletions(-) diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..dd714610b 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -180,6 +180,13 @@ disable_full_debug() { sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH" } +restart_containerd() { + sudo systemctl restart containerd + if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then + die "containerd seems not operational after restarted" + fi +} + # Configure containerd for confidential containers. Among other things, it ensures # the CRI handler is configured to deal with confidential container. # @@ -198,6 +205,7 @@ configure_cc_containerd() { # installed via operator it will assume containerd is in right state # already. [ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0 + sudo iptables -w -P FORWARD ACCEPT # Even if we are not saving the original file it is a good idea to # restart containerd because it might be in an inconsistent state here. @@ -205,8 +213,7 @@ configure_cc_containerd() { sleep 5 [ -n "$saved_containerd_conf_file" ] && \ sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file" - sudo systemctl start containerd - waitForProcess 30 5 "sudo crictl info >/dev/null" + restart_containerd # Ensure the cc CRI handler is set. local cri_handler=$(sudo crictl info | \ @@ -223,11 +230,7 @@ configure_cc_containerd() { sudo tee -a "$containerd_conf_file" fi - sudo systemctl restart containerd - if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then - die "containerd seems not operational after reconfigured" - fi - sudo iptables -w -P FORWARD ACCEPT + restart_containerd } # diff --git a/integration/containerd/confidential/tests_common.sh b/integration/containerd/confidential/tests_common.sh index c44f08bf6..12e6862ca 100644 --- a/integration/containerd/confidential/tests_common.sh +++ b/integration/containerd/confidential/tests_common.sh @@ -25,8 +25,8 @@ setup_common() { echo "Prepare containerd for Confidential Container" SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" - configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + restart_containerd # Note: ensure that intructions changing the kernel parameters are # executed *after* saving the original list. saved_kernel_params=$(get_kernel_params) diff --git a/integration/kubernetes/confidential/agent_image.bats b/integration/kubernetes/confidential/agent_image.bats index de5c815e8..1b789a981 100644 --- a/integration/kubernetes/confidential/agent_image.bats +++ b/integration/kubernetes/confidential/agent_image.bats @@ -31,7 +31,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}" test_tag="[cc][agent][kubernetes][containerd]" setup() { - setup_common + setup_containerd + restart_containerd + reconfigure_kata } @test "$test_tag Test can launch pod with measured boot enabled" { diff --git a/integration/kubernetes/confidential/sealed_secret.bats b/integration/kubernetes/confidential/sealed_secret.bats index 282005547..0b0ef0b81 100644 --- a/integration/kubernetes/confidential/sealed_secret.bats +++ b/integration/kubernetes/confidential/sealed_secret.bats @@ -22,13 +22,8 @@ setup() { SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" - echo "Reconfigure Kata Containers" - switch_image_service_offload on - clear_kernel_params - add_kernel_params "${original_kernel_params}" - - setup_proxy - switch_measured_rootfs_verity_scheme none + restart_containerd + reconfigure_kata kubectl delete secret sealed-secret --ignore-not-found # Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault diff --git a/integration/kubernetes/confidential/sev.bats b/integration/kubernetes/confidential/sev.bats index 9f515a8bb..49f2a5ad2 100644 --- a/integration/kubernetes/confidential/sev.bats +++ b/integration/kubernetes/confidential/sev.bats @@ -25,6 +25,7 @@ load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${TESTS_REPO_DIR}/lib/common.bash" load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh" load "${TESTS_REPO_DIR}/integration/kubernetes/confidential/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" # Delete all test services k8s_delete_all() { @@ -54,7 +55,7 @@ setup_file() { # Configure CoCo settings in containerd config local saved_containerd_conf_file="/etc/containerd/config.toml.$$" - configure_cc_containerd "${saved_containerd_conf_file}" + restart_containerd # KBS setup and run echo "Setting up simple-kbs..." diff --git a/integration/kubernetes/confidential/snp.bats b/integration/kubernetes/confidential/snp.bats index 3f03ebda2..6367a4d60 100644 --- a/integration/kubernetes/confidential/snp.bats +++ b/integration/kubernetes/confidential/snp.bats @@ -21,6 +21,7 @@ export SSH_KEY_FILE load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${TESTS_REPO_DIR}/lib/common.bash" load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" # Delete all test services k8s_delete_all() { @@ -37,7 +38,7 @@ setup_file() { # Configure CoCo settings in containerd config local saved_containerd_conf_file="/etc/containerd/config.toml.$$" - configure_cc_containerd "${saved_containerd_conf_file}" + restart_containerd # Pull unencrypted image and retrieve ssh keys echo "Pulling unencrypted image and retrieve ssh key..." diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh index 177ed450e..f447c14fb 100644 --- a/integration/kubernetes/confidential/tests_common.sh +++ b/integration/kubernetes/confidential/tests_common.sh @@ -25,11 +25,21 @@ setup_common() { pod_id="" kubernetes_delete_all_cc_pods_if_any_exists || true +} + +# Setup containerd for tests. +# +setup_containerd() { + setup_common echo "Prepare containerd for Confidential Container" SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" +} +# Reconfigure Kata for tests +# +reconfigure_kata() { echo "Reconfigure Kata Containers" switch_image_service_offload on clear_kernel_params @@ -58,6 +68,14 @@ teardown_common() { add_kernel_params "${original_kernel_params}" switch_image_service_offload off disable_full_debug + + # Restore containerd to pre-test state. + if [ -f "$SAVED_CONTAINERD_CONF_FILE" ]; then + systemctl stop containerd || true + sleep 5 + mv -f "$SAVED_CONTAINERD_CONF_FILE" "/etc/containerd/config.toml" + systemctl start containerd || true + fi } @@ -97,8 +115,9 @@ create_test_pod() { new_pod_config() { local base_config="${FIXTURES_DIR}/pod-config.yaml.in" local image="$1" + local index="${2:-}" local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") - IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config" + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$index" envsubst < "$base_config" > "$new_config" echo "$new_config" }