From edb4cc1095210e0ee00c838b53f60eadfc55586a Mon Sep 17 00:00:00 2001
From: Homayoon Alimohammadi <homayoon.alimohammadi@canonical.com>
Date: Tue, 27 Aug 2024 10:50:25 +0400
Subject: [PATCH] Add k8s endpoint check to markNodeReady (#615)

Add k8s endpoint check to markNodeReady to fix csrsigning controller bug
---
 src/k8s/pkg/client/kubernetes/status.go | 10 ++++++++
 src/k8s/pkg/k8sd/app/app.go             | 32 +++++++++++++++++--------
 src/k8s/pkg/k8sd/app/hooks_start.go     |  7 +++++-
 3 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/src/k8s/pkg/client/kubernetes/status.go b/src/k8s/pkg/client/kubernetes/status.go
index b09c7352f..e98f87778 100644
--- a/src/k8s/pkg/client/kubernetes/status.go
+++ b/src/k8s/pkg/client/kubernetes/status.go
@@ -20,6 +20,16 @@ func (c *Client) WaitKubernetesEndpointAvailable(ctx context.Context) error {
 	})
 }
 
+func (c *Client) CheckKubernetesEndpoint(ctx context.Context) error {
+	if endpoint, err := c.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{}); err != nil {
+		return fmt.Errorf("failed to get kubernetes endpoint: %w", err)
+	} else if endpoint == nil {
+		return fmt.Errorf("kubernetes endpoint not found")
+	}
+
+	return nil
+}
+
 // HasReadyNodes checks the status of all nodes in the Kubernetes cluster.
 // HasReadyNodes returns true if there is at least one Ready node in the cluster, false otherwise.
 func (c *Client) HasReadyNodes(ctx context.Context) (bool, error) {
diff --git a/src/k8s/pkg/k8sd/app/app.go b/src/k8s/pkg/k8sd/app/app.go
index 944cda197..ad472a371 100644
--- a/src/k8s/pkg/k8sd/app/app.go
+++ b/src/k8s/pkg/k8sd/app/app.go
@@ -15,6 +15,7 @@ import (
 	"github.com/canonical/k8s/pkg/k8sd/database"
 	"github.com/canonical/k8s/pkg/log"
 	"github.com/canonical/k8s/pkg/snap"
+	"github.com/canonical/k8s/pkg/utils/control"
 	"github.com/canonical/microcluster/v2/client"
 	"github.com/canonical/microcluster/v2/microcluster"
 	"github.com/canonical/microcluster/v2/state"
@@ -228,17 +229,28 @@ func (a *App) Run(ctx context.Context, customHooks *state.Hooks) error {
 	return nil
 }
 
-func (a *App) markNodeReady(ctx context.Context, s state.State) {
-	for {
-		if err := s.Database().IsOpen(ctx); err == nil {
-			a.readyWg.Done()
-			return
-		}
+func (a *App) markNodeReady(ctx context.Context, s state.State) error {
+	// wait for the database to be open
+	if err := control.WaitUntilReady(ctx, func() (bool, error) {
+		return s.Database().IsOpen(ctx) == nil, nil
+	}); err != nil {
+		return fmt.Errorf("failed to wait for database to be open: %w", err)
+	}
 
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(3 * time.Second):
+	// check kubernetes endpoint
+	if err := control.WaitUntilReady(ctx, func() (bool, error) {
+		client, err := a.snap.KubernetesNodeClient("")
+		if err != nil {
+			return false, nil
+		}
+		if err := client.CheckKubernetesEndpoint(ctx); err != nil {
+			return false, nil
 		}
+		return true, nil
+	}); err != nil {
+		return fmt.Errorf("failed to wait for kubernetes endpoint: %w", err)
 	}
+
+	a.readyWg.Done()
+	return nil
 }
diff --git a/src/k8s/pkg/k8sd/app/hooks_start.go b/src/k8s/pkg/k8sd/app/hooks_start.go
index 0277f74f0..4c90b86f0 100644
--- a/src/k8s/pkg/k8sd/app/hooks_start.go
+++ b/src/k8s/pkg/k8sd/app/hooks_start.go
@@ -10,6 +10,7 @@ import (
 	"github.com/canonical/k8s/pkg/k8sd/database"
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/types"
+	"github.com/canonical/k8s/pkg/log"
 	"github.com/canonical/k8s/pkg/utils"
 	pkiutil "github.com/canonical/k8s/pkg/utils/pki"
 	"github.com/canonical/microcluster/v2/state"
@@ -17,7 +18,11 @@ import (
 
 func (a *App) onStart(ctx context.Context, s state.State) error {
 	// start a goroutine to mark the node as running
-	go a.markNodeReady(ctx, s)
+	go func() {
+		if err := a.markNodeReady(ctx, s); err != nil {
+			log.FromContext(ctx).Error(err, "Failed to mark node as ready")
+		}
+	}()
 
 	// start node config controller
 	if a.nodeConfigController != nil {