Skip to content

Commit

Permalink
test-review
Browse files Browse the repository at this point in the history
  • Loading branch information
camilamacedo86 committed Sep 3, 2024
1 parent f867ac0 commit 5d772aa
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 392 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"time"

. "github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -124,59 +123,47 @@ var _ = Describe("Manager", Ordered, func() {
EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
})

// TODO(user): Customize the e2e test suite to include
// additional scenarios specific to your project.

It("should ensure the metrics endpoint is serving metrics", func() {
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
"--clusterrole=project-metrics-reader",
fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName),
)
_, err := utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")
Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")

By("validating that the metrics service is available")
cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace)
_, err = utils.Run(cmd)
ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist")
Expect(err).NotTo(HaveOccurred(), "Metrics service should exist")

By("validating that the ServiceMonitor for Prometheus is applied in the namespace")
cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace)
_, err = utils.Run(cmd)
ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist")
Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist")

By("getting the service account token")
token, err := serviceAccountToken()
ExpectWithOffset(2, err).NotTo(HaveOccurred())
ExpectWithOffset(2, token).NotTo(BeEmpty())
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())

By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func() error {
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
output, err := utils.Run(cmd)
if err != nil {
return err
}
if !strings.Contains(string(output), "8443") {
return fmt.Errorf("metrics endpoint is not ready")
}
return nil
g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information")
g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
}
EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed())
Eventually(verifyMetricsEndpointReady).WithTimeout(2 * time.Minute).Should(Succeed())

By("verifying that the controller manager is serving the metrics server")
Eventually(func() error {
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
logs, err := utils.Run(cmd)
if err != nil {
return err
}
if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") {
return fmt.Errorf("metrics server not yet started")
}
return nil
}, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server")
g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs")
g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), "Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).WithTimeout(2 * time.Minute).Should(Succeed())

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
Expand All @@ -185,26 +172,22 @@ var _ = Describe("Manager", Ordered, func() {
"--", "/bin/sh", "-c", fmt.Sprintf(
"curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics",
token, metricsServiceName, namespace))
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod")
Expect(utils.Run(cmd)).To(Succeed(), "Failed to create curl-metrics pod")

By("waiting for the curl-metrics pod to complete.")
verifyCurlUp := func() error {
verifyCurlUp := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pods", "curl-metrics",
"-o", "jsonpath={.status.phase}",
"-n", namespace)
status, err := utils.Run(cmd)
ExpectWithOffset(3, err).NotTo(HaveOccurred())
if string(status) != "Succeeded" {
return fmt.Errorf("curl pod in %s status", status)
}
return nil
g.Expect(err).NotTo(HaveOccurred())
g.Expect(status).To(Equal("Succeeded"), fmt.Sprintf("curl pod in %s status", status))
}
EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed())
Eventually(verifyCurlUp).WithTimeout(5 * time.Minute).Should(Succeed())

By("getting the metrics by checking curl-metrics logs")
metricsOutput := getMetricsOutput()
ExpectWithOffset(1, metricsOutput).To(ContainSubstring(
Expect(metricsOutput).To(ContainSubstring(
"controller_runtime_reconcile_total",
))
})
Expand All @@ -213,7 +196,7 @@ var _ = Describe("Manager", Ordered, func() {
// Consider applying sample/CR(s) and check their status and/or verifying
// the reconciliation by using the metrics, i.e.:
// metricsOutput := getMetricsOutput()
// ExpectWithOffset(1, metricsOutput).To(ContainSubstring(
// Expect(metricsOutput).To(ContainSubstring(
// fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
// strings.ToLower(<Kind>),
// ))
Expand All @@ -239,7 +222,7 @@ func serviceAccountToken() (string, error) {

var out string
var rawJson string
Eventually(func() error {
verifyTokenCreation := func(g Gomega) {
// Execute kubectl command to create the token
cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf(
"/api/v1/namespaces/%s/serviceaccounts/%s/token",
Expand All @@ -248,22 +231,18 @@ func serviceAccountToken() (string, error) {
), "-f", tokenRequestFile)

output, err := cmd.CombinedOutput()
if err != nil {
return err
}
g.Expect(err).NotTo(HaveOccurred())

rawJson = string(output)

// Parse the JSON output to extract the token
var token tokenRequest
err = json.Unmarshal([]byte(rawJson), &token)
if err != nil {
return err
}
g.Expect(err).NotTo(HaveOccurred())

out = token.Status.Token
return nil
}, time.Minute, time.Second).Should(Succeed())
}
Eventually(verifyTokenCreation).WithTimeout(time.Minute).Should(Succeed())

return out, err
}
Expand All @@ -273,9 +252,9 @@ func getMetricsOutput() string {
By("getting the curl-metrics logs")
cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
metricsOutput, err := utils.Run(cmd)
ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
metricsOutputStr := string(metricsOutput)
ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK"))
Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK"))
return metricsOutputStr
}

Expand Down
77 changes: 28 additions & 49 deletions docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"time"

. "github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -124,59 +123,47 @@ var _ = Describe("Manager", Ordered, func() {
EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
})

// TODO(user): Customize the e2e test suite to include
// additional scenarios specific to your project.

It("should ensure the metrics endpoint is serving metrics", func() {
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
"--clusterrole=project-metrics-reader",
fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName),
)
_, err := utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")
Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")

By("validating that the metrics service is available")
cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace)
_, err = utils.Run(cmd)
ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist")
Expect(err).NotTo(HaveOccurred(), "Metrics service should exist")

By("validating that the ServiceMonitor for Prometheus is applied in the namespace")
cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace)
_, err = utils.Run(cmd)
ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist")
Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist")

By("getting the service account token")
token, err := serviceAccountToken()
ExpectWithOffset(2, err).NotTo(HaveOccurred())
ExpectWithOffset(2, token).NotTo(BeEmpty())
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())

By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func() error {
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
output, err := utils.Run(cmd)
if err != nil {
return err
}
if !strings.Contains(string(output), "8443") {
return fmt.Errorf("metrics endpoint is not ready")
}
return nil
g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information")
g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
}
EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed())
Eventually(verifyMetricsEndpointReady).WithTimeout(2 * time.Minute).Should(Succeed())

By("verifying that the controller manager is serving the metrics server")
Eventually(func() error {
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
logs, err := utils.Run(cmd)
if err != nil {
return err
}
if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") {
return fmt.Errorf("metrics server not yet started")
}
return nil
}, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server")
g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs")
g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), "Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).WithTimeout(2 * time.Minute).Should(Succeed())

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
Expand All @@ -185,26 +172,22 @@ var _ = Describe("Manager", Ordered, func() {
"--", "/bin/sh", "-c", fmt.Sprintf(
"curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics",
token, metricsServiceName, namespace))
_, err = utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod")
Expect(utils.Run(cmd)).To(Succeed(), "Failed to create curl-metrics pod")

By("waiting for the curl-metrics pod to complete.")
verifyCurlUp := func() error {
verifyCurlUp := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pods", "curl-metrics",
"-o", "jsonpath={.status.phase}",
"-n", namespace)
status, err := utils.Run(cmd)
ExpectWithOffset(3, err).NotTo(HaveOccurred())
if string(status) != "Succeeded" {
return fmt.Errorf("curl pod in %s status", status)
}
return nil
g.Expect(err).NotTo(HaveOccurred())
g.Expect(status).To(Equal("Succeeded"), fmt.Sprintf("curl pod in %s status", status))
}
EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed())
Eventually(verifyCurlUp).WithTimeout(5 * time.Minute).Should(Succeed())

By("getting the metrics by checking curl-metrics logs")
metricsOutput := getMetricsOutput()
ExpectWithOffset(1, metricsOutput).To(ContainSubstring(
Expect(metricsOutput).To(ContainSubstring(
"controller_runtime_reconcile_total",
))
})
Expand All @@ -213,7 +196,7 @@ var _ = Describe("Manager", Ordered, func() {
// Consider applying sample/CR(s) and check their status and/or verifying
// the reconciliation by using the metrics, i.e.:
// metricsOutput := getMetricsOutput()
// ExpectWithOffset(1, metricsOutput).To(ContainSubstring(
// Expect(metricsOutput).To(ContainSubstring(
// fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
// strings.ToLower(<Kind>),
// ))
Expand All @@ -239,7 +222,7 @@ func serviceAccountToken() (string, error) {

var out string
var rawJson string
Eventually(func() error {
verifyTokenCreation := func(g Gomega) {
// Execute kubectl command to create the token
cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf(
"/api/v1/namespaces/%s/serviceaccounts/%s/token",
Expand All @@ -248,22 +231,18 @@ func serviceAccountToken() (string, error) {
), "-f", tokenRequestFile)

output, err := cmd.CombinedOutput()
if err != nil {
return err
}
g.Expect(err).NotTo(HaveOccurred())

rawJson = string(output)

// Parse the JSON output to extract the token
var token tokenRequest
err = json.Unmarshal([]byte(rawJson), &token)
if err != nil {
return err
}
g.Expect(err).NotTo(HaveOccurred())

out = token.Status.Token
return nil
}, time.Minute, time.Second).Should(Succeed())
}
Eventually(verifyTokenCreation).WithTimeout(time.Minute).Should(Succeed())

return out, err
}
Expand All @@ -273,9 +252,9 @@ func getMetricsOutput() string {
By("getting the curl-metrics logs")
cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
metricsOutput, err := utils.Run(cmd)
ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
metricsOutputStr := string(metricsOutput)
ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK"))
Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK"))
return metricsOutputStr
}

Expand Down
Loading

0 comments on commit 5d772aa

Please sign in to comment.