diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go index 141ebe9d5f4..265d1829d2c 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go index 2812ef5b973..f447d2c1a31 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "tutorial.kubebuilder.io/project/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go +++ b/docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go index 2f582ed11f9..5e3806b98ee 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go index 66f7dcef8fd..cef5c44db8b 100644 --- a/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go +++ b/docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "example.com/memcached/test/utils" ) +// namespace where the project is deployed in const namespace = "project-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/docs/book/src/getting-started/testdata/project/test/utils/utils.go b/docs/book/src/getting-started/testdata/project/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/docs/book/src/getting-started/testdata/project/test/utils/utils.go +++ b/docs/book/src/getting-started/testdata/project/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go index b34f9d855eb..5334efdfbbe 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/suite.go @@ -83,6 +83,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go index 6288f9d4cfd..2ffee5d71b2 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go @@ -44,44 +44,54 @@ var TestTemplate = `{{ .Boilerplate }} package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "{{ .Repo }}/test/utils" ) +// namespace where the project is deployed in const namespace = "{{ .ProjectName }}-system" +// serviceAccountName created for the project +const serviceAccountName = "{{ .ProjectName }}-controller-manager" +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "{{ .ProjectName }}-controller-manager-metrics-service" +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "{{ .ProjectName }}-metrics-binding" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -93,13 +103,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -111,32 +123,162 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole={{ .ProjectName}}-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5 * time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(` + "`controller_runtime_reconcile_total{controller=\"%s\",result=\"success\"} 1`" + `, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = ` + "`" + `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + "`" + ` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string ` + "`json:\"token\"`" + ` + } ` + "`json:\"status\"`" + ` +} ` diff --git a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go index f34d5cfbbc7..fd68964be6f 100644 --- a/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go +++ b/pkg/plugins/golang/v4/scaffolds/internal/templates/test/utils/utils.go @@ -42,6 +42,8 @@ var utilsTemplate = `{{ .Boilerplate }} package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -223,4 +225,54 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} ` diff --git a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go index ae4e19086c6..2404cc6ea74 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go index fe3a6e4d75d..fbf91bee6cb 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go +++ b/testdata/project-v4-multigroup-with-deploy-image/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-multigroup-with-deploy-image/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-multigroup-with-deploy-image-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-multigroup-with-deploy-image-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-multigroup-with-deploy-image-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-multigroup-with-deploy-image-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-multigroup-with-deploy-image-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go b/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go +++ b/testdata/project-v4-multigroup-with-deploy-image/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go b/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go index b68ec10b389..be68fdc2df9 100644 --- a/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-multigroup/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-multigroup/test/e2e/e2e_test.go b/testdata/project-v4-multigroup/test/e2e/e2e_test.go index 580c8745b03..06c0f49519e 100644 --- a/testdata/project-v4-multigroup/test/e2e/e2e_test.go +++ b/testdata/project-v4-multigroup/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-multigroup/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-multigroup-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-multigroup-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-multigroup-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-multigroup-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-multigroup-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-multigroup/test/utils/utils.go b/testdata/project-v4-multigroup/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-multigroup/test/utils/utils.go +++ b/testdata/project-v4-multigroup/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go b/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go index 68830827b0b..c2020988060 100644 --- a/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-with-deploy-image/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go b/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go index 144c8c76d1d..5375a53b929 100644 --- a/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go +++ b/testdata/project-v4-with-deploy-image/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-with-deploy-image/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-with-deploy-image-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-with-deploy-image-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-with-deploy-image-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-with-deploy-image-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-with-deploy-image-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-with-deploy-image/test/utils/utils.go b/testdata/project-v4-with-deploy-image/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-with-deploy-image/test/utils/utils.go +++ b/testdata/project-v4-with-deploy-image/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go b/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go index f006e422db7..24357e32ee0 100644 --- a/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4-with-grafana/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4-with-grafana/test/e2e/e2e_test.go b/testdata/project-v4-with-grafana/test/e2e/e2e_test.go index a26e626abbe..c5b96a92c33 100644 --- a/testdata/project-v4-with-grafana/test/e2e/e2e_test.go +++ b/testdata/project-v4-with-grafana/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4-with-grafana/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-with-grafana-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-with-grafana-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-with-grafana-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-with-grafana-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-with-grafana-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4-with-grafana/test/utils/utils.go b/testdata/project-v4-with-grafana/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4-with-grafana/test/utils/utils.go +++ b/testdata/project-v4-with-grafana/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +} diff --git a/testdata/project-v4/test/e2e/e2e_suite_test.go b/testdata/project-v4/test/e2e/e2e_suite_test.go index 048651805bc..59afad48a06 100644 --- a/testdata/project-v4/test/e2e/e2e_suite_test.go +++ b/testdata/project-v4/test/e2e/e2e_suite_test.go @@ -57,6 +57,9 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { + By("Ensure that Prometheus is enabled") + _ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#") + By("generating files") cmd := exec.Command("make", "generate") _, err := utils.Run(cmd) diff --git a/testdata/project-v4/test/e2e/e2e_test.go b/testdata/project-v4/test/e2e/e2e_test.go index 690c711c470..1300dcbe96e 100644 --- a/testdata/project-v4/test/e2e/e2e_test.go +++ b/testdata/project-v4/test/e2e/e2e_test.go @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,34 +30,44 @@ import ( "sigs.k8s.io/kubebuilder/testdata/project-v4/test/utils" ) +// namespace where the project is deployed in const namespace = "project-v4-system" -// Define a set of end-to-end (e2e) tests to validate the behavior of the controller. -var _ = Describe("controller", Ordered, func() { +// serviceAccountName created for the project +const serviceAccountName = "project-v4-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "project-v4-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "project-v4-metrics-binding" + +var _ = Describe("Manager", Ordered, func() { // Before running the tests, set up the environment by creating the namespace, // installing CRDs, and deploying the controller. BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create namespace") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create namespace") By("installing CRDs") cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to install CRDs") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, // and deleting the namespace. AfterAll(func() { + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + cmd = exec.Command("make", "undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -66,13 +79,15 @@ var _ = Describe("controller", Ordered, func() { _, _ = utils.Run(cmd) }) - // The Context block contains the actual tests that validate the operator's behavior. - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + // The Context block contains the actual tests that validate the manager's behavior. + Context("Manager", func() { + var controllerPodName string + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { + verifyControllerUp := func(g Gomega) { // Get the name of the controller-manager pod cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", @@ -84,31 +99,161 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expected 1 controller pod running, but got %d", len(podNames)) - } + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod status") - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Running"), "Incorrect controller-manager pod status") } // Repeatedly check if the controller-manager pod is running until it succeeds or times out. - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=project-v4-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("validating that the ServiceMonitor for Prometheus is applied in the namespace") + cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "ServiceMonitor should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve endpoints information") + g.Expect(string(output)).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + logs, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller manager logs") + g.Expect(string(logs)).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + Expect(utils.Run(cmd)).Error().NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + g.Expect(utils.Run(cmd)).To(BeEquivalentTo("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) - // TODO(user): Customize the e2e test suite to include - // additional scenarios specific to your project. + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + var rawJson string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + rawJson = string(output) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + metricsOutputStr := string(metricsOutput) + Expect(metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutputStr +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/testdata/project-v4/test/utils/utils.go b/testdata/project-v4/test/utils/utils.go index 777825397c5..db4ad3f02f6 100644 --- a/testdata/project-v4/test/utils/utils.go +++ b/testdata/project-v4/test/utils/utils.go @@ -17,6 +17,8 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" @@ -198,3 +200,52 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +}