From ec2f58d4b268667cb42346cb3630262a18ded108 Mon Sep 17 00:00:00 2001 From: utam0k Date: Sat, 6 May 2023 12:02:18 +0000 Subject: [PATCH 1/2] Add easy way to test with K8s Signed-off-by: utam0k --- .gitignore | 2 +- Makefile | 27 ++++++++++++- .../src/process/container_init_process.rs | 4 +- .../process/container_intermediate_process.rs | 3 +- crates/libcontainer/src/process/mod.rs | 2 +- crates/libcontainer/src/rootfs/mount.rs | 2 +- crates/libcontainer/src/seccomp/mod.rs | 2 +- hack/debug.bt | 38 ++++++++++++++++++- tests/k8s/deploy.yaml | 27 +++++++++++++ 9 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 tests/k8s/deploy.yaml diff --git a/.gitignore b/.gitignore index 1836f8791..0dd74a3f7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ tags.temp *~ /bundle.tar.gz -/test.log \ No newline at end of file +/test.log diff --git a/Makefile b/Makefile index bd0095d04..012c7420c 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,32 @@ test-oci: oci-tests rust-oci-tests .PHONY: test-all test-all: unittest featuretest oci-tests containerd-test # currently not doing rust-oci here +.PHONY: test/k3s +test/k3s: bin/k3s + sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo 'default_runtime_name = "youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo ' runtime_type = "io.containerd.runc.v2"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo ' [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki.options]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo ' BinaryName = "$(PWD)/youki"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \ + echo "CONTAINERD_NAMESPACE='default'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env && \ + echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env && \ + sudo systemctl daemon-reload && \ + sudo systemctl restart k3s-youki && \ + sudo bin/k3s kubectl apply -f tests/k8s/deploy.yaml + sudo bin/k3s kubectl wait deployment nginx-deployment --for condition=Available=True --timeout=90s && \ + sudo bin/k3s kubectl get pods -o wide + +.PHONY: test/k3s/clean +test/k3s/clean: + sudo bin/k3s-youki-uninstall.sh + # Misc +# +.PHONY: bin/k3s +bin/k3s: + mkdir -p bin && \ + curl -sfL https://get.k3s.io | INSTALL_K3S_BIN_DIR=$(PWD)/bin INSTALL_K3S_SYMLINK=skip INSTALL_K3S_NAME=youki sh - .PHONY: lint lint: @@ -69,7 +94,7 @@ lint: .PHONY: hack/bpftrace hack/bpftrace: - BPFTRACE_STRLEN=120 ./hack/debug.bt + BPFTRACE_STRLEN=125 ./hack/debug.bt .PHONY: clean clean: diff --git a/crates/libcontainer/src/process/container_init_process.rs b/crates/libcontainer/src/process/container_init_process.rs index dd6b210fb..7024e46e6 100644 --- a/crates/libcontainer/src/process/container_init_process.rs +++ b/crates/libcontainer/src/process/container_init_process.rs @@ -204,7 +204,7 @@ pub fn container_init_process( // before pivot_root is called. This runs in the container namespaces. if let Some(hooks) = hooks { hooks::run_hooks(hooks.create_container().as_ref(), container) - .context("Failed to run create container hooks")?; + .context("failed to run create container hooks")?; } let bind_service = namespaces.get(LinuxNamespaceType::User).is_some(); @@ -216,7 +216,7 @@ pub fn container_init_process( bind_service, namespaces.get(LinuxNamespaceType::Cgroup).is_some(), ) - .with_context(|| "Failed to prepare rootfs")?; + .with_context(|| "failed to prepare rootfs")?; // Entering into the rootfs jail. If mount namespace is specified, then // we use pivot_root, but if we are on the host mount namespace, we will diff --git a/crates/libcontainer/src/process/container_intermediate_process.rs b/crates/libcontainer/src/process/container_intermediate_process.rs index aabdba447..7d606c431 100644 --- a/crates/libcontainer/src/process/container_intermediate_process.rs +++ b/crates/libcontainer/src/process/container_intermediate_process.rs @@ -119,7 +119,8 @@ pub fn container_intermediate_process( write(exec_notify_fd, buf.as_bytes())?; close(exec_notify_fd)?; } - Err(ProcessError::InitProcessFailed) + log::error!("failed to initialize container process: {e}"); + Err(ProcessError::InitProcessFailed { msg: e.to_string() }) } } })?; diff --git a/crates/libcontainer/src/process/mod.rs b/crates/libcontainer/src/process/mod.rs index 5962b6c01..94bf6bf2f 100644 --- a/crates/libcontainer/src/process/mod.rs +++ b/crates/libcontainer/src/process/mod.rs @@ -24,7 +24,7 @@ pub enum ProcessError { child_name: String, }, #[error("failed init process")] - InitProcessFailed, + InitProcessFailed { msg: String }, #[error("failed intermediate process")] IntermediateProcessFailed, #[error("io error: {0}")] diff --git a/crates/libcontainer/src/rootfs/mount.rs b/crates/libcontainer/src/rootfs/mount.rs index 418baee89..07fa80fcd 100644 --- a/crates/libcontainer/src/rootfs/mount.rs +++ b/crates/libcontainer/src/rootfs/mount.rs @@ -48,7 +48,7 @@ impl Mount { } pub fn setup_mount(&self, mount: &SpecMount, options: &MountOptions) -> Result<()> { - log::debug!("Mounting {:?}", mount); + log::debug!("mounting {:?}", mount); let mut mount_option_config = parse_mount(mount); match mount.typ().as_deref() { diff --git a/crates/libcontainer/src/seccomp/mod.rs b/crates/libcontainer/src/seccomp/mod.rs index d6bb4e166..c06071d54 100644 --- a/crates/libcontainer/src/seccomp/mod.rs +++ b/crates/libcontainer/src/seccomp/mod.rs @@ -136,7 +136,7 @@ pub fn initialize_seccomp(seccomp: &LinuxSeccomp) -> Result> { // When the action is the same as the default action, the rule is redundant. We can // skip this here to avoid failing when we add the rules. log::warn!( - "Detect a seccomp action that is the same as the default action: {:?}", + "detect a seccomp action that is the same as the default action: {:?}", syscall ); continue; diff --git a/hack/debug.bt b/hack/debug.bt index b80288326..6b1043300 100755 --- a/hack/debug.bt +++ b/hack/debug.bt @@ -41,9 +41,45 @@ tracepoint:syscalls:sys_exit_openat tracepoint:syscalls:sys_enter_clone3 /comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ { - printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "clone3"); + printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "clone3"); } +tracepoint:syscalls:sys_enter_setns +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setns"); + printf("fd=%d, flag=%d\n", args->fd, args->flags); +} + +tracepoint:syscalls:sys_enter_capset +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s\n", elapsed , comm, pid, "capset"); +} + +tracepoint:syscalls:sys_enter_pivot_root +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "pivt_root"); + printf("new_root=%s, put_old=%s\n", str(args->new_root), str(args->put_old)); +} + +tracepoint:syscalls:sys_enter_mount +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "mount"); + printf("dev_name=%s, dir_name=%s\n", str(args->dev_name), str(args->dir_name)); +} + +tracepoint:syscalls:sys_enter_setresuid +/comm == "4"|| comm == "youki" || comm == "youki:[1:INTER]" || comm == "youki:[2:INIT]"/ +{ + printf("%-12ld %15s %-8d %-9s ", elapsed , comm, pid, "setresuid"); + printf("ruid=%d, euid=%d, suid=%d\n", args->ruid, args->euid, args->suid); +} + + + END { clear(@filename); diff --git a/tests/k8s/deploy.yaml b/tests/k8s/deploy.yaml new file mode 100644 index 000000000..6fe094e5b --- /dev/null +++ b/tests/k8s/deploy.yaml @@ -0,0 +1,27 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: youki +handler: youki +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + selector: + matchLabels: + app: nginx + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + runtimeClassName: youki + containers: + - name: nginx + image: nginx:1.16.1 + ports: + - containerPort: 80 + automountServiceAccountToken: false From 29cf869c6835043fd311fd93e2e2920e1a486c65 Mon Sep 17 00:00:00 2001 From: utam0k Date: Sat, 6 May 2023 23:15:32 +0000 Subject: [PATCH 2/2] Add the handly script for kind and youki Signed-off-by: utam0k --- .github/scripts/dependency.sh | 4 +-- .../workflows/benchmark_execution_time.yml | 4 +-- ...ainerd_integration_tests.yaml => e2e.yaml} | 22 ++++++++++-- .../integration_tests_validation.yaml | 2 +- .github/workflows/main.yml | 6 ++-- .github/workflows/podman_tests.yaml | 2 +- .github/workflows/release.yml | 4 +-- .gitignore | 2 ++ Makefile | 32 +++++++++++++++-- .../src/process/container_init_process.rs | 4 +-- tests/k8s/Dockerfile | 36 +++++++++++++++++++ 11 files changed, 101 insertions(+), 17 deletions(-) rename .github/workflows/{containerd_integration_tests.yaml => e2e.yaml} (73%) create mode 100644 tests/k8s/Dockerfile diff --git a/.github/scripts/dependency.sh b/.github/scripts/dependency.sh index e57330fcb..b8c55e2dc 100755 --- a/.github/scripts/dependency.sh +++ b/.github/scripts/dependency.sh @@ -1,4 +1,4 @@ #!/bin/bash -sudo apt-get -y update -sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev +apt-get -y update +apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev diff --git a/.github/workflows/benchmark_execution_time.yml b/.github/workflows/benchmark_execution_time.yml index 5d319b5b8..9ef0964fc 100644 --- a/.github/workflows/benchmark_execution_time.yml +++ b/.github/workflows/benchmark_execution_time.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v3 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 @@ -41,7 +41,7 @@ jobs: ref: main - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 diff --git a/.github/workflows/containerd_integration_tests.yaml b/.github/workflows/e2e.yaml similarity index 73% rename from .github/workflows/containerd_integration_tests.yaml rename to .github/workflows/e2e.yaml index 03f399c3d..cf9f2e1d7 100644 --- a/.github/workflows/containerd_integration_tests.yaml +++ b/.github/workflows/e2e.yaml @@ -1,4 +1,4 @@ -name: 🧪 Containerd integration test +name: 🧪 e2e test on: push: @@ -17,7 +17,7 @@ jobs: - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Build youki run: make youki-release - name: Upload youki binary @@ -57,3 +57,21 @@ jobs: runc --version - name: Integration Test run: sudo make TEST_RUNTIME=io.containerd.runc.v2 TESTFLAGS="-timeout 40m" integration + # + # k8s-tests: + # runs-on: ubuntu-22.04 + # needs: [youki-build] + # timeout-minutes: 40 + # steps: + # - uses: actions/checkout@v3 + # - name: Download youki binary + # uses: actions/download-artifact@v3 + # with: + # name: youki + # - name: Add the permission to run + # run: chmod +x ./youki + # - name: test/k8s/deploy + # run: make test/k8s/deploy + # # - name: Debug + # # if: ${{ always() }} + # # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/integration_tests_validation.yaml b/.github/workflows/integration_tests_validation.yaml index 17b2571b9..8b285f3d4 100644 --- a/.github/workflows/integration_tests_validation.yaml +++ b/.github/workflows/integration_tests_validation.yaml @@ -31,7 +31,7 @@ jobs: - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Install runc 1.1.0 run: | wget -q https://github.com/opencontainers/runc/releases/download/v1.1.0/runc.amd64 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 417b945d5..525fe203b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 - run: rustup component add rustfmt clippy - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - run: sudo apt-get install -y pkg-config libsystemd-dev libdbus-glib-1-dev libelf-dev libseccomp-dev - name: Check formatting run: cargo fmt --all -- --check @@ -84,7 +84,7 @@ jobs: with: tool: cargo-llvm-cov@0.4.0 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Run Test Coverage for youki run: | cargo llvm-cov clean --workspace @@ -105,7 +105,7 @@ jobs: - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - uses: actions/setup-go@v3 with: go-version: "1.17.6" diff --git a/.github/workflows/podman_tests.yaml b/.github/workflows/podman_tests.yaml index d9ec7c02a..e4680c4ca 100644 --- a/.github/workflows/podman_tests.yaml +++ b/.github/workflows/podman_tests.yaml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - run: make youki-dev - run: sudo cp youki /usr/local/bin - name: Install requirements for Podman diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 49d009ef9..5f8d09db1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: with: submodules: recursive - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 - name: Build @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install requirements - run: ./.github/scripts/dependency.sh + run: sudo ./.github/scripts/dependency.sh - name: Setup Rust toolchain and cache uses: actions-rust-lang/setup-rust-toolchain@v1.3.7 - name: Release build diff --git a/.gitignore b/.gitignore index 0dd74a3f7..fdeed4a22 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ tags.temp /bundle.tar.gz /test.log + +/tests/k8s/_out/ diff --git a/Makefile b/Makefile index 012c7420c..899b9dcd6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ ROOT = $(shell git rev-parse --show-toplevel) +DOCKER_BUILD ?= docker buildx build + +KIND_CLUSTER_NAME ?= youki + # builds .PHONY:build @@ -80,13 +84,37 @@ test/k3s: bin/k3s test/k3s/clean: sudo bin/k3s-youki-uninstall.sh -# Misc -# +.PHONY: test/k8s/cluster +test/k8s/cluster: bin/kind tests/k8s/_out/img bin/kind + bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat tests/k8s/_out/img)" + +.PHONY: test/k8s/deploy +test/k8s/deploy: test/k8s/cluster + kubectl --context=kind-$(KIND_CLUSTER_NAME) apply -f tests/k8s/deploy.yaml + kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment nginx-deployment --for condition=Available=True --timeout=90s + kubectl --context=kind-$(KIND_CLUSTER_NAME) get pods -o wide + +# Bin + .PHONY: bin/k3s bin/k3s: mkdir -p bin && \ curl -sfL https://get.k3s.io | INSTALL_K3S_BIN_DIR=$(PWD)/bin INSTALL_K3S_SYMLINK=skip INSTALL_K3S_NAME=youki sh - +.PHONY: bin/kind +bin/kind: tests/k8s/Dockerfile + $(DOCKER_BUILD) --output=bin/ -f tests/k8s/Dockerfile --target kind-bin . + +.PHONY: test/k8s/clean +test/k8s/clean: + kind delete cluster --name $(KIND_CLUSTER_NAME) + rm -r tests/k8s/_out + +tests/k8s/_out/img: tests/k8s/Dockerfile Cargo.toml Cargo.lock $(shell find . -type f -name '*.rs') + mkdir -p $(@D) && $(DOCKER_BUILD) -f tests/k8s/Dockerfile --iidfile=$(@) --load . + +# Misc + .PHONY: lint lint: cargo fmt --all -- --check diff --git a/crates/libcontainer/src/process/container_init_process.rs b/crates/libcontainer/src/process/container_init_process.rs index 7024e46e6..05d02c37a 100644 --- a/crates/libcontainer/src/process/container_init_process.rs +++ b/crates/libcontainer/src/process/container_init_process.rs @@ -376,9 +376,9 @@ pub fn container_init_process( warn!("seccomp not available, unable to enforce no_new_privileges!") } - capabilities::reset_effective(syscall).context("Failed to reset effective capabilities")?; + capabilities::reset_effective(syscall).context("failed to reset effective capabilities")?; if let Some(caps) = proc.capabilities() { - capabilities::drop_privileges(caps, syscall).context("Failed to drop capabilities")?; + capabilities::drop_privileges(caps, syscall).context("failed to drop capabilities")?; } // Change directory to process.cwd if process.cwd is not empty diff --git a/tests/k8s/Dockerfile b/tests/k8s/Dockerfile new file mode 100644 index 000000000..eacb6e9fb --- /dev/null +++ b/tests/k8s/Dockerfile @@ -0,0 +1,36 @@ +# syntax=docker/dockerfile:1.4 + +ARG KIND_NODE_VERSION=v1.23.13 + +FROM kindest/node:${KIND_NODE_VERSION} AS kind-base + +FROM kind-base AS shim-build +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > /tmp/rustup.sh && sh /tmp/rustup.sh -y --profile=minimal +ENV PATH="/root/.cargo/bin:${PATH}" +WORKDIR /shim +COPY ./youki /shim/youki + +FROM scratch AS shim +COPY --from=shim-build /shim/youki / + +FROM kind-base AS kind-fetch +ARG TARGETARCH +ARG KIND_VERSION=v0.17.0 +RUN curl -sSLf https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-${TARGETARCH} > /root/kind && chmod +x /root/kind + +FROM scratch AS kind-bin +COPY --from=kind-fetch /root/kind /kind + +FROM kind-base +RUN <> /etc/containerd/config.toml +echo ' runtime_type = "io.containerd.runc.v2"' >> /etc/containerd/config.toml +echo ' [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.youki.options]' >> /etc/containerd/config.toml +echo ' BinaryName = "/usr/local/bin/youki"' >> /etc/containerd/config.toml +sed -i 's,SystemdCgroup = true,,' /etc/containerd/config.toml +EOF +COPY .github/scripts/dependency.sh .github/scripts/dependency.sh +RUN apt-get update && .github/scripts/dependency.sh +COPY --link --from=shim /* /usr/local/bin/ +