From ab536a4e008b7469eafd12c6e643f16c96f2fb6d Mon Sep 17 00:00:00 2001 From: Taichi Takemura Date: Wed, 8 Nov 2023 15:56:15 +0900 Subject: [PATCH] Add e2e test (#9) * Add e2e test Signed-off-by: zeroalphat --------- Signed-off-by: zeroalphat --- .github/workflows/ci.yaml | 22 +++++-- Dockerfile.cli | 18 ++++++ Dockerfile.daemon | 37 ++++++++++++ Makefile | 9 +++ Makefile.versions | 2 + config/namespace/kustomization.yaml | 2 + config/namespace/namespace.yaml | 4 ++ config/rbac/kustomization.yaml | 4 ++ config/rbac/role.yaml | 19 ++++++ config/rbac/rolebinding.yaml | 27 +++++++++ config/rbac/serviceaccount.yaml | 5 ++ e2e/Makefile | 86 +++++++++++++++++++++++++++ e2e/Makefile.versions | 3 + e2e/e2e_test.go | 21 +++++++ e2e/env_test.go | 7 +++ e2e/kind-config.yaml | 5 ++ e2e/manifests/necoperf-client.yaml | 15 +++++ e2e/manifests/necoperf-daemonset.yaml | 61 +++++++++++++++++++ e2e/manifests/profiled-pod.yaml | 12 ++++ e2e/run_test.go | 24 ++++++++ e2e/suite_test.go | 74 +++++++++++++++++++++++ 21 files changed, 452 insertions(+), 5 deletions(-) create mode 100644 Dockerfile.cli create mode 100644 Dockerfile.daemon create mode 100644 config/namespace/kustomization.yaml create mode 100644 config/namespace/namespace.yaml create mode 100644 config/rbac/kustomization.yaml create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/rolebinding.yaml create mode 100644 config/rbac/serviceaccount.yaml create mode 100644 e2e/Makefile create mode 100644 e2e/Makefile.versions create mode 100644 e2e/e2e_test.go create mode 100644 e2e/env_test.go create mode 100644 e2e/kind-config.yaml create mode 100644 e2e/manifests/necoperf-client.yaml create mode 100644 e2e/manifests/necoperf-daemonset.yaml create mode 100644 e2e/manifests/profiled-pod.yaml create mode 100644 e2e/run_test.go create mode 100644 e2e/suite_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 832d9cf..e898178 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,15 +4,27 @@ on: push: branches: - 'main' -env: - go-version: "1.21" jobs: test: name: Small tests runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: - go-version: ${{ env.go-version }} + go-version-file: go.mod - run: make test + e2e: + name: End-to-end tests + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + - run: make setup + working-directory: e2e + - run: make start + working-directory: e2e + - run: make test + working-directory: e2e diff --git a/Dockerfile.cli b/Dockerfile.cli new file mode 100644 index 0000000..a299919 --- /dev/null +++ b/Dockerfile.cli @@ -0,0 +1,18 @@ +FROM ghcr.io/cybozu/golang:1.21-jammy as builder +WORKDIR /work +COPY go.mod go.mod +COPY go.sum go.sum + +COPY cmd/necoperf-cli cmd/necoperf-cli +COPY internal internal +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o necoperf-cli ./cmd/necoperf-cli + +FROM quay.io/cybozu/pause:3.9 as pause +FROM ghcr.io/cybozu/ubuntu:22.04 +LABEL org.opencontainers.image.source https://github.com/cybozu-go/necoperf + +COPY --from=pause /pause /usr/local/bin/pause +COPY --from=builder /work/necoperf-cli /usr/local/bin/necoperf-cli + +USER 1000:1000 +CMD ["pause"] diff --git a/Dockerfile.daemon b/Dockerfile.daemon new file mode 100644 index 0000000..35bb109 --- /dev/null +++ b/Dockerfile.daemon @@ -0,0 +1,37 @@ +ARG FLATCAR_VERSION +FROM ghcr.io/cybozu/golang:1.21-jammy as builder + +WORKDIR /work +COPY go.mod go.mod +COPY go.sum go.sum + +COPY cmd/necoperf-daemon cmd/necoperf-daemon +COPY internal internal +RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o necoperf-daemon ./cmd/necoperf-daemon + +FROM ghcr.io/flatcar/flatcar-sdk-amd64:${FLATCAR_VERSION} as flatcar +RUN ldd /usr/bin/perf | cut -d" " -f3 | xargs tar --dereference -cf libs.tar + +FROM scratch +LABEL org.opencontainers.image.source https://github.com/cybozu-go/necoperf +ENV HOME=/home/necoperf +COPY --from=flatcar /bin/ /bin/ +COPY --from=flatcar /sbin/setcap /sbin/setcap +COPY --from=flatcar /lib64 /lib64 +COPY --from=flatcar /usr/bin/perf /usr/bin/sleep /usr/bin/ + +# install perf dependency +COPY --from=flatcar /libs.tar /libs.tar +RUN mkdir -p /usr/lib64 \ + && tar -xf libs.tar \ + && rm libs.tar + +COPY --from=builder /work/necoperf-daemon /usr/local/bin/necoperf-daemon + +RUN setcap "cap_perfmon,cap_sys_ptrace,cap_syslog,cap_sys_admin,cap_sys_chroot=ep" /usr/bin/perf \ + && mkdir -p ${HOME} \ + && chown -R 1000:1000 ${HOME} +WORKDIR ${HOME} + +USER 1000:1000 +ENTRYPOINT ["/usr/local/bin/necoperf-daemon", "start"] diff --git a/Makefile b/Makefile index 7835736..7a34c22 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,15 @@ internal/rpc/necoperf_grpc.pb.go: internal/rpc/necoperf.proto docs/necoperf-grpc.md: internal/rpc/necoperf.proto $(PROTOC) --doc_out=docs --doc_opt=markdown,$@ $< +.PHONY: docker-build +docker-build: build + docker build -t necoperf-daemon:dev --build-arg="FLATCAR_VERSION=$(FLATCAR_VERSION)" -f Dockerfile.daemon . + docker build -t necoperf-cli:dev -f Dockerfile.cli . + +.PHONY: e2e +e2e: + $(MAKE) -C e2e + ##@ Tools .PHONY: setup diff --git a/Makefile.versions b/Makefile.versions index 06a16d4..14b57c9 100644 --- a/Makefile.versions +++ b/Makefile.versions @@ -1 +1,3 @@ E2ETEST_K8S_VERSION := 1.27.1 +# ref to https://github.com/orgs/flatcar/packages/container/package/flatcar-sdk-amd64 +FLATCAR_VERSION := 3602.0.0 diff --git a/config/namespace/kustomization.yaml b/config/namespace/kustomization.yaml new file mode 100644 index 0000000..736967b --- /dev/null +++ b/config/namespace/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - namespace.yaml diff --git a/config/namespace/namespace.yaml b/config/namespace/namespace.yaml new file mode 100644 index 0000000..7174193 --- /dev/null +++ b/config/namespace/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: necoperf diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 0000000..7daccbb --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - serviceaccount.yaml + - role.yaml + - rolebinding.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 0000000..f4abe22 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: necoperf-daemon-role + namespace: necoperf +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get","list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: necoperf-cli-role + namespace: default +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get","list"] diff --git a/config/rbac/rolebinding.yaml b/config/rbac/rolebinding.yaml new file mode 100644 index 0000000..bde6315 --- /dev/null +++ b/config/rbac/rolebinding.yaml @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: necoperf-cli-rolebinding + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: necoperf-cli-role +subjects: +- kind: ServiceAccount + name: necoperf-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: necoperf-daemon-rolebinding + namespace: necoperf +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: necoperf-daemon-role +subjects: +- kind: ServiceAccount + name: necoperf-sa + namespace: default diff --git a/config/rbac/serviceaccount.yaml b/config/rbac/serviceaccount.yaml new file mode 100644 index 0000000..94f8909 --- /dev/null +++ b/config/rbac/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: necoperf-sa + namespace: default diff --git a/e2e/Makefile b/e2e/Makefile new file mode 100644 index 0000000..16269a4 --- /dev/null +++ b/e2e/Makefile @@ -0,0 +1,86 @@ +include Makefile.versions + +ARCH ?= amd64 +OS ?= linux + +PROJECT_DIR := $(CURDIR)/../ +BIN_DIR := $(PROJECT_DIR)/bin + +CURL := curl -sSLf +KUBECTL := $(BIN_DIR)/kubectl +KUSTOMIZE := $(BIN_DIR)/kustomize + +KIND := $(BIN_DIR)/kind +KIND_CLUSTER_NAME := necoperf +KIND_CONFIG := kind-config.yaml + +export KUBECONFIG + +.PHONY: help +help: + @echo "Choose one of the following target" + @echo + @echo "setup Setup tools" + @echo "start Start kind cluster and install accurate" + @echo "test Run e2e tests" + @echo "logs Save logs as logs.tar.gz" + @echo "stop Stop the kind cluster" + +.PHONY: setup +setup: kubectl kustomize kind + +.PHONY: start +start: + $(KIND) create cluster --name=$(KIND_CLUSTER_NAME) --config=$(KIND_CONFIG) --image=kindest/node:v$(E2ETEST_K8S_VERSION) --wait 1m + $(MAKE) -C ../ docker-build + $(KIND) load docker-image necoperf-daemon:dev --name=$(KIND_CLUSTER_NAME) + $(KIND) load docker-image necoperf-cli:dev --name=$(KIND_CLUSTER_NAME) + $(KUSTOMIZE) build ../config/namespace | $(KUBECTL) apply -f - + $(KUSTOMIZE) build ../config/rbac | $(KUBECTL) apply -f - + +.PHONY: test +test: + env RUN_E2E=1 \ + go test -v -race . -ginkgo.v -ginkgo.fail-fast + +.PHONY: stop +stop: + $(KIND) delete cluster --name=$(KIND_CLUSTER_NAME) + -docker image rm necoperf-daemon:dev + -docker image rm necoperf-cli:dev + -docker image prune -f + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) +$(KUSTOMIZE): $(KUSTOMIZE)-$(KUSTOMIZE_VERSION) + ln -sf $(notdir $<) $@ + +$(KUSTOMIZE)-$(KUSTOMIZE_VERSION): + mkdir -p $(dir $@) + curl -fsL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv$(KUSTOMIZE_VERSION)/kustomize_v$(KUSTOMIZE_VERSION)_linux_amd64.tar.gz | \ + tar -xzf - -O > $@ + chmod a+x $@ + +.PHONY: kind +kind: $(KIND) +$(KIND): $(KIND)-$(KIND_VERSION) + ln -sf $(notdir $<) $@ + +$(KIND)-$(KIND_VERSION): + mkdir -p $(dir $@) + $(CURL) -o $@ https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-$(OS)-$(ARCH) + chmod a+x $@ + +.PHONY: kubectl +kubectl: $(KUBECTL) +$(KUBECTL): $(KUBECTL)-$(E2ETEST_K8S_VERSION) + ln -sf $(notdir $<) $@ + +$(KUBECTL)-$(E2ETEST_K8S_VERSION): + mkdir -p $(dir $@) + $(CURL) -o $@ https://storage.googleapis.com/kubernetes-release/release/v$(E2ETEST_K8S_VERSION)/bin/$(OS)/$(ARCH)/kubectl + chmod a+x $@ + +.PHONY: clean +clean: + rm -rf $(BIN_DIR) diff --git a/e2e/Makefile.versions b/e2e/Makefile.versions new file mode 100644 index 0000000..a68bf92 --- /dev/null +++ b/e2e/Makefile.versions @@ -0,0 +1,3 @@ +E2ETEST_K8S_VERSION := 1.27.1 +KIND_VERSION := 0.20.0 +KUSTOMIZE_VERSION := 5.1.0 diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 0000000..92faf97 --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,21 @@ +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("necoperf e2e test", func() { + It("should be able to run necoperf-cli", func() { + By("executing necoperf-cli") + _, err := kubectl(nil, "exec", "necoperf-client", "--", + "necoperf-cli", "profile", "profiled-pod", + "--timeout", "100s") + Expect(err).NotTo(HaveOccurred()) + + By("checking if profiling result is created") + out, err := kubectl(nil, "exec", "necoperf-client", "--", "cat", "/tmp/profiled-pod.script") + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(ContainSubstring("yes")) + }) +}) diff --git a/e2e/env_test.go b/e2e/env_test.go new file mode 100644 index 0000000..ea826f6 --- /dev/null +++ b/e2e/env_test.go @@ -0,0 +1,7 @@ +package e2e + +import "os" + +var ( + runE2E = os.Getenv("RUN_E2E") != "" +) diff --git a/e2e/kind-config.yaml b/e2e/kind-config.yaml new file mode 100644 index 0000000..ead21eb --- /dev/null +++ b/e2e/kind-config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker diff --git a/e2e/manifests/necoperf-client.yaml b/e2e/manifests/necoperf-client.yaml new file mode 100644 index 0000000..45d6268 --- /dev/null +++ b/e2e/manifests/necoperf-client.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: necoperf-client + namespace: default +spec: + containers: + - name: necoperf-client + image: necoperf-cli:dev + imagePullPolicy: IfNotPresent + command: ["pause"] + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + serviceAccountName: necoperf-sa diff --git a/e2e/manifests/necoperf-daemonset.yaml b/e2e/manifests/necoperf-daemonset.yaml new file mode 100644 index 0000000..d9941f0 --- /dev/null +++ b/e2e/manifests/necoperf-daemonset.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: necoperf-daemon + namespace: necoperf + labels: + k8s-app: necoperf-daemon +spec: + selector: + matchLabels: + app.kubernetes.io/name: necoperf-daemon + template: + metadata: + labels: + app.kubernetes.io/name: necoperf-daemon + spec: + hostPID: true + terminationGracePeriodSeconds: 30 + containers: + - name: necoperf-daemon + image: necoperf-daemon:dev + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + add: + - SYSLOG + - SYS_PTRACE + - PERFMON + - SYS_CHROOT + - SYS_ADMIN + drop: + - ALL + volumeMounts: + - name: necoperf-workdir + mountPath: /var/necoperf + - name: containerd-sock + mountPath: /run/containerd/containerd.sock + - name: sys-kernel-tracing + mountPath: /sys/kernel/tracing + ports: + - name: grpc + containerPort: 6543 + - name: metrics + containerPort: 6541 + livenessProbe: + grpc: + port: 6543 + initialDelaySeconds: 10 + readinessProbe: + grpc: + port: 6543 + initialDelaySeconds: 5 + volumes: + - name: necoperf-workdir + emptyDir: {} + - name: containerd-sock + hostPath: + path: /run/containerd/containerd.sock + - name: sys-kernel-tracing + hostPath: + path: /sys/kernel/tracing diff --git a/e2e/manifests/profiled-pod.yaml b/e2e/manifests/profiled-pod.yaml new file mode 100644 index 0000000..b4623b5 --- /dev/null +++ b/e2e/manifests/profiled-pod.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: profiled-pod +spec: + securityContext: + runAsUser: 10000 + runAsGroup: 10000 + containers: + - name: profiled-pod + image: ghcr.io/cybozu/ubuntu:22.04 + command: ["yes"] diff --git a/e2e/run_test.go b/e2e/run_test.go new file mode 100644 index 0000000..42d5218 --- /dev/null +++ b/e2e/run_test.go @@ -0,0 +1,24 @@ +package e2e + +import ( + "bytes" + "fmt" + "os/exec" +) + +func kubectl(input []byte, args ...string) ([]byte, error) { + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + cmd := exec.Command("../bin/kubectl", args...) + cmd.Stdout = stdout + cmd.Stderr = stderr + if input != nil { + cmd.Stdin = bytes.NewReader(input) + } + err := cmd.Run() + if err == nil { + return stdout.Bytes(), nil + } + + return nil, fmt.Errorf("kubectl failed with %s: stderr=%s", err, stderr) +} diff --git a/e2e/suite_test.go b/e2e/suite_test.go new file mode 100644 index 0000000..79a5026 --- /dev/null +++ b/e2e/suite_test.go @@ -0,0 +1,74 @@ +package e2e + +import ( + "encoding/json" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" +) + +const ( + daemonsetDesiredNumber = 1 +) + +func TestE2E(t *testing.T) { + if !runE2E { + t.Skip("no RUN_E2E environment variable") + } + + RegisterFailHandler(Fail) + SetDefaultEventuallyTimeout(1 * time.Minute) + SetDefaultEventuallyPollingInterval(1 * time.Second) + RunSpecs(t, "E2E Suite") +} + +var _ = BeforeSuite(func() { + By("prepare pod for profiling") + _, err := kubectl(nil, "apply", "-f", "./manifests/profiled-pod.yaml") + Expect(err).NotTo(HaveOccurred()) + + By("waiting for profiled-pod to be ready") + Eventually(func(g Gomega) { + res, err := kubectl(nil, "get", "pod", "profiled-pod", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + pod := corev1.Pod{} + err = json.Unmarshal(res, &pod) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) + }).Should(Succeed()) + + By("creating necoperf-cli pod from manifests") + _, err = kubectl(nil, "apply", "-f", "./manifests/necoperf-client.yaml") + Expect(err).NotTo(HaveOccurred()) + + By("waiting for necoperf-cli pod to be ready") + Eventually(func(g Gomega) { + res, err := kubectl(nil, "get", "pod", "necoperf-client", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + + pod := corev1.Pod{} + err = json.Unmarshal(res, &pod) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) + }).Should(Succeed()) + + By("creating necoperf daemonset from manifest") + _, err = kubectl(nil, "apply", "-f", "./manifests/necoperf-daemonset.yaml") + Expect(err).NotTo(HaveOccurred()) + + By("waiting for necoperf-daemon daemonnset to be ready") + Eventually(func(g Gomega) { + res, err := kubectl(nil, "get", "daemonset", "necoperf-daemon", "-n", "necoperf", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + daemonset := appsv1.DaemonSet{} + err = json.Unmarshal(res, &daemonset) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(int(daemonset.Status.NumberAvailable)).To(Equal(daemonsetDesiredNumber)) + g.Expect(int(daemonset.Status.UpdatedNumberScheduled)).To(Equal(daemonsetDesiredNumber)) + g.Expect(int(daemonset.Status.NumberReady)).To(Equal(daemonsetDesiredNumber)) + }, 3*time.Minute).Should(Succeed()) +})