Ensure binaries do not hang during bootstrap (#495)

* ensure binaries do not hang during bootstrap

* run binaries as part of pre-init checks

src/k8s/pkg/k8sd/app/hooks_bootstrap.go

@@ -179,6 +179,12 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon
 			K8sdPublicKey: utils.Pointer(response.K8sdPublicKey),
 		},
 	}
+
+	// Pre-init checks
+	if err := snap.PreInitChecks(s.Context, cfg); err != nil {
+		return fmt.Errorf("pre-init checks failed for worker node: %w", err)
+	}
+
 	if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
 		if _, err := database.SetClusterConfig(ctx, tx, cfg); err != nil {
 			return fmt.Errorf("failed to write cluster configuration: %w", err)
@@ -339,6 +345,11 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
 	cfg.Certificates.K8sdPublicKey = utils.Pointer(certificates.K8sdPublicKey)
 	cfg.Certificates.K8sdPrivateKey = utils.Pointer(certificates.K8sdPrivateKey)
 
+	// Pre-init checks
+	if err := snap.PreInitChecks(s.Context, cfg); err != nil {
+		return fmt.Errorf("pre-init checks failed for bootstrap node: %w", err)
+	}
+
 	// Generate kubeconfigs
 	if err := setupKubeconfigs(s, snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), *certificates); err != nil {
 		return fmt.Errorf("failed to generate kubeconfigs: %w", err)

src/k8s/pkg/k8sd/app/hooks_join.go

@@ -109,6 +109,13 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
 	if err := certificates.CompleteCertificates(); err != nil {
 		return fmt.Errorf("failed to initialize control plane certificates: %w", err)
 	}
+
+	// Pre-init checks
+	if err := snap.PreInitChecks(s.Context, cfg); err != nil {
+		return fmt.Errorf("pre-init checks failed for joining node: %w", err)
+	}
+
+	// Write certificates to disk
 	if _, err := setup.EnsureControlPlanePKI(snap, certificates); err != nil {
 		return fmt.Errorf("failed to write control plane certificates: %w", err)
 	}

src/k8s/pkg/snap/interface.go

@@ -6,6 +6,7 @@ import (
 	"github.com/canonical/k8s/pkg/client/dqlite"
 	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/client/kubernetes"
+	"github.com/canonical/k8s/pkg/k8sd/types"
 )
 
 // Snap abstracts file system paths and interacting with the k8s services.
@@ -54,4 +55,6 @@ type Snap interface {
 	HelmClient() helm.Client // admin helm client
 
 	K8sDqliteClient(ctx context.Context) (*dqlite.Client, error) // go-dqlite client for k8s-dqlite
+
+	PreInitChecks(ctx context.Context, config types.ClusterConfig) error // pre-init checks before k8s-snap can start
 }

src/k8s/pkg/snap/mock/mock.go

@@ -7,6 +7,7 @@ import (
 	"github.com/canonical/k8s/pkg/client/dqlite"
 	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/client/kubernetes"
+	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 )
 
@@ -56,6 +57,9 @@ type Snap struct {
 	SnapctlGetCalledWith [][]string
 	SnapctlGetErr        error
 
+	PreInitChecksCalledWith []types.ClusterConfig
+	PreInitChecksErr        error
+
 	Mock Mock
 }
 
@@ -173,5 +177,9 @@ func (s *Snap) SnapctlSet(ctx context.Context, args ...string) error {
 	s.SnapctlSetCalledWith = append(s.SnapctlGetCalledWith, args)
 	return s.SnapctlSetErr
 }
+func (s *Snap) PreInitChecks(ctx context.Context, config types.ClusterConfig) error {
+	s.PreInitChecksCalledWith = append(s.PreInitChecksCalledWith, config)
+	return s.PreInitChecksErr
+}
 
 var _ snap.Snap = &Snap{}

src/k8s/pkg/snap/snap.go

@@ -12,6 +12,7 @@ import (
 	"github.com/canonical/k8s/pkg/client/dqlite"
 	"github.com/canonical/k8s/pkg/client/helm"
 	"github.com/canonical/k8s/pkg/client/kubernetes"
+	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/utils"
 	"github.com/moby/sys/mountinfo"
 	"gopkg.in/yaml.v2"
@@ -238,4 +239,18 @@ func (s *snap) SnapctlSet(ctx context.Context, args ...string) error {
 	return s.runCommand(ctx, append([]string{"snapctl", "set"}, args...))
 }
 
+func (s *snap) PreInitChecks(ctx context.Context, config types.ClusterConfig) error {
+	// TODO: check for available ports for k8s-dqlite, apiserver, containerd, etc
+
+	// NOTE(neoaggelos): in some environments the Kubernetes might hang when running for the first time
+	// This works around the issue by running them once during the install hook
+	for _, binary := range []string{"kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy", "kubelet"} {
+		if err := s.runCommand(ctx, []string{filepath.Join(s.snapDir, "bin", binary), "--version"}); err != nil {
+			return fmt.Errorf("%q binary could not run: %w", binary, err)
+		}
+	}
+
+	return nil
+}
+
 var _ Snap = &snap{}