Detect how the vcluster was put to sleep and guide the user to the co…

…rrect commands to wake it up

Warns allows waking and adding a sleeping helm vcluster in a single command.
Adds the ability to set chart dir for local dev when adding external cluster.
Only errors if helm driver is specified, otherwise it falls back to a platform resume.
Remove old check due to platform's auto conversion of helm slept clusters.
Continue with adding the secret in either case and only add if they choose to wake the cluster.

Co-authored-by: Russell Centanni <russell.centanni@gmail.com>

cmd/vclusterctl/cmd/platform/add/cluster.go

@@ -143,7 +143,7 @@ func (cmd *ClusterCmd) Run(ctx context.Context, args []string) error {
 
 	if os.Getenv("DEVELOPMENT") == "true" {
 		helmArgs = []string{
-			"upgrade", "--install", "loft", "./chart",
+			"upgrade", "--install", "loft", cmp.Or(os.Getenv("DEVELOPMENT_CHART_DIR"), "./chart"),
 			"--create-namespace",
 			"--namespace", namespace,
 			"--set", "agentOnly=true",

cmd/vclusterctl/cmd/resume.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"cmp"
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/loft-sh/log"
@@ -73,5 +74,14 @@ func (cmd *ResumeCmd) Run(ctx context.Context, args []string) error {
 		return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log)
 	}
 
-	return cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log)
+	if err := cli.ResumeHelm(ctx, cmd.GlobalFlags, args[0], cmd.Log); err != nil {
+		// If they specified a driver, don't fall back to the platform automatically.
+		if cmd.Driver == "" && errors.Is(err, cli.ErrPlatformDriverRequired) {
+			return cli.ResumePlatform(ctx, &cmd.ResumeOptions, cfg, args[0], cmd.Log)
+		}
+
+		return err
+	}
+
+	return nil
 }

config/config.go

@@ -225,6 +225,14 @@ func (c *Config) Distro() string {
 	return K8SDistro
 }
 
+func (c *Config) IsConfiguredForSleepMode() bool {
+	if c != nil && c.External["platform"] == nil {
+		return false
+	}
+
+	return c.External["platform"]["autoSleep"] != nil || c.External["platform"]["autoDelete"] != nil
+}
+
 // ValidateChanges checks for disallowed config changes.
 // Currently only certain backingstore changes are allowed but no distro change.
 func ValidateChanges(oldCfg, newCfg *Config) error {

pkg/cli/add_vcluster_helm.go

@@ -3,12 +3,16 @@ package cli
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/loft-sh/log"
+	"github.com/loft-sh/log/survey"
 	"github.com/loft-sh/vcluster/pkg/cli/find"
 	"github.com/loft-sh/vcluster/pkg/cli/flags"
 	"github.com/loft-sh/vcluster/pkg/lifecycle"
 	"github.com/loft-sh/vcluster/pkg/platform"
+	"github.com/loft-sh/vcluster/pkg/platform/clihelper"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -44,6 +48,46 @@ func AddVClusterHelm(
 		return err
 	}
 
+	snoozed := false
+	// If the vCluster was paused with the helm driver, adding it to the platform will only create the secret for registration
+	// which leads to confusing behavior for the user since they won't see the cluster in the platform UI until it is resumed.
+	if lifecycle.IsPaused(vCluster) {
+		log.Infof("vCluster %s is currently sleeping. It will not be added to the platform until it wakes again.", vCluster.Name)
+
+		snoozeConfirmation := "No. Leave it sleeping. (It will be added automatically on next wakeup)"
+		answer, err := log.Question(&survey.QuestionOptions{
+			Question:     fmt.Sprintf("Would you like to wake vCluster %s now to add immediately?", vCluster.Name),
+			DefaultValue: snoozeConfirmation,
+			Options: []string{
+				snoozeConfirmation,
+				"Yes. Wake and add now.",
+			},
+		})
+
+		if err != nil {
+			return fmt.Errorf("failed to capture your response %w", err)
+		}
+
+		if snoozed = answer == snoozeConfirmation; !snoozed {
+			if err = ResumeHelm(ctx, globalFlags, vClusterName, log); err != nil {
+				return fmt.Errorf("failed to wake up vCluster %s: %w", vClusterName, err)
+			}
+
+			err = wait.PollUntilContextTimeout(ctx, time.Second, clihelper.Timeout(), false, func(ctx context.Context) (done bool, err error) {
+				vCluster, err = find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log)
+				if err != nil {
+					return false, err
+				}
+
+				return !lifecycle.IsPaused(vCluster), nil
+			})
+
+			if err != nil {
+				return fmt.Errorf("error waiting for vCluster to wake up %w", err)
+			}
+		}
+	}
+
 	// apply platform secret
 	err = platform.ApplyPlatformSecret(
 		ctx,
@@ -68,6 +112,11 @@ func AddVClusterHelm(
 		}
 	}
 
-	log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name)
+	if snoozed {
+		log.Infof("vCluster %s/%s will be added the next time it awakes", vCluster.Namespace, vCluster.Name)
+		log.Donef("Run 'vcluster wakeup --help' to learn how to wake up vCluster %s/%s to complete the add operation.", vCluster.Namespace, vCluster.Name)
+	} else {
+		log.Donef("Successfully added vCluster %s/%s", vCluster.Namespace, vCluster.Name)
+	}
 	return nil
 }

pkg/cli/create_helm.go

@@ -268,7 +268,7 @@ func CreateHelm(ctx context.Context, options *CreateOptions, globalFlags *flags.
 		cmd.Connect = false
 	}
 
-	if isSleepModeConfigured(vClusterConfig) {
+	if vClusterConfig.IsConfiguredForSleepMode() {
 		if agentDeployed, err := cmd.isLoftAgentDeployed(ctx); err != nil {
 			return fmt.Errorf("is agent deployed: %w", err)
 		} else if !agentDeployed {
@@ -391,13 +391,6 @@ func (cmd *createHelm) isLoftAgentDeployed(ctx context.Context) (bool, error) {
 	return len(podList.Items) > 0, nil
 }
 
-func isSleepModeConfigured(vClusterConfig *config.Config) bool {
-	if vClusterConfig == nil || vClusterConfig.External == nil || vClusterConfig.External["platform"] == nil {
-		return false
-	}
-	return vClusterConfig.External["platform"]["autoSleep"] != nil || vClusterConfig.External["platform"]["autoDelete"] != nil
-}
-
 func isVClusterDeployed(release *helm.Release) bool {
 	return release != nil &&
 		release.Chart != nil &&

pkg/cli/find/find.go

@@ -11,6 +11,7 @@ import (
 	"github.com/loft-sh/log/survey"
 	"github.com/loft-sh/log/terminal"
 	"github.com/loft-sh/vcluster/pkg/platform"
+	"github.com/loft-sh/vcluster/pkg/platform/sleepmode"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/loft-sh/vcluster/pkg/constants"
@@ -31,6 +32,8 @@ type VCluster struct {
 	Created       metav1.Time
 	Name          string
 	Namespace     string
+	Annotations   map[string]string
+	Labels        map[string]string
 	Status        Status
 	Context       string
 	Version       string
@@ -161,6 +164,20 @@ func GetVCluster(ctx context.Context, context, name, namespace string, log log.L
 	return nil, fmt.Errorf("unexpected error searching for selected virtual cluster")
 }
 
+func (v *VCluster) IsSleeping() bool {
+	return sleepmode.IsSleeping(v)
+}
+
+// GetAnnotations implements Annotated
+func (v *VCluster) GetAnnotations() map[string]string {
+	return v.Annotations
+}
+
+// GetLabels implements Labeled
+func (v *VCluster) GetLabels() map[string]string {
+	return v.Labels
+}
+
 func FormatOptions(format string, options [][]string) []string {
 	if len(options) == 0 {
 		return []string{}
@@ -312,12 +329,7 @@ func findInContext(ctx context.Context, context, name, namespace string, timeout
 				continue
 			}
 
-			var paused string
-
-			if p.Annotations != nil {
-				paused = p.Annotations[constants.PausedAnnotation]
-			}
-			if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && paused != "true" {
+			if p.Spec.Replicas != nil && *p.Spec.Replicas == 0 && !isPaused(&p) {
 				// if the stateful set has been scaled down we'll ignore it -- this happens when
 				// using devspace to do vcluster plugin dev for example, devspace scales down the
 				// vcluster stateful set and re-creates a deployment for "dev mode" so we end up
@@ -413,6 +425,8 @@ func getVCluster(ctx context.Context, object client.Object, context, release str
 	return VCluster{
 		Name:          release,
 		Namespace:     namespace,
+		Annotations:   object.GetAnnotations(),
+		Labels:        object.GetLabels(),
 		Status:        Status(status),
 		Created:       created,
 		Context:       context,
@@ -561,3 +575,10 @@ func GetPodStatus(pod *corev1.Pod) string {
 	}
 	return reason
 }
+
+func isPaused(v client.Object) bool {
+	annotations := v.GetAnnotations()
+	labels := v.GetLabels()
+
+	return annotations[constants.PausedAnnotation] == "true" || labels[sleepmode.Label] == "true"
+}

pkg/cli/pause_helm.go

@@ -30,6 +30,11 @@ func PauseHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName
 		return err
 	}
 
+	if vCluster.IsSleeping() {
+		log.Infof("vcluster %s/%s is already sleeping", globalFlags.Namespace, vClusterName)
+		return nil
+	}
+
 	err = lifecycle.PauseVCluster(ctx, kubeClient, vClusterName, globalFlags.Namespace, log)
 	if err != nil {
 		return err

pkg/cli/pause_platform.go

@@ -22,11 +22,15 @@ func PausePlatform(ctx context.Context, options *PauseOptions, cfg *cliconfig.CL
 	if err != nil {
 		return err
 	}
+
 	vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log)
 	if err != nil {
 		return err
-	} else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External {
-		return fmt.Errorf("cannot pause a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag")
+	}
+
+	if vCluster.IsInstanceSleeping() {
+		log.Infof("vcluster %s/%s is already paused", vCluster.VirtualCluster.Namespace, vClusterName)
+		return nil
 	}
 
 	managementClient, err := platformClient.Management()

pkg/cli/resume_helm.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/loft-sh/log"
@@ -17,12 +18,18 @@ type ResumeOptions struct {
 	Project string
 }
 
+var ErrPlatformDriverRequired = errors.New("cannot resume a virtual cluster that is paused by the platform, please run 'vcluster use driver platform' or use the '--driver platform' flag")
+
 func ResumeHelm(ctx context.Context, globalFlags *flags.GlobalFlags, vClusterName string, log log.Logger) error {
 	vCluster, err := find.GetVCluster(ctx, globalFlags.Context, vClusterName, globalFlags.Namespace, log)
 	if err != nil {
 		return err
 	}
 
+	if vCluster.IsSleeping() {
+		return ErrPlatformDriverRequired
+	}
+
 	kubeClient, err := prepareResume(vCluster, globalFlags)
 	if err != nil {
 		return err

pkg/cli/resume_platform.go

@@ -18,8 +18,14 @@ func ResumePlatform(ctx context.Context, options *ResumeOptions, config *config.
 	vCluster, err := find.GetPlatformVCluster(ctx, platformClient, vClusterName, options.Project, log)
 	if err != nil {
 		return err
-	} else if vCluster.VirtualCluster != nil && vCluster.VirtualCluster.Spec.External {
-		return fmt.Errorf("cannot resume a virtual cluster that was created via helm, please run 'vcluster use driver helm' or use the '--driver helm' flag")
+	}
+
+	if !vCluster.IsInstanceSleeping() {
+		return fmt.Errorf(
+			"couldn't find a paused vcluster %s in namespace %s. Make sure the vcluster exists and was paused previously",
+			vCluster.VirtualCluster.Spec.ClusterRef.VirtualCluster,
+			vCluster.VirtualCluster.Spec.ClusterRef.Namespace,
+		)
 	}
 
 	managementClient, err := platformClient.Management()

pkg/kube/meta.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/loft-sh/log"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
@@ -17,7 +16,30 @@ const (
 	LoftCustomLinksDelimiter = "\n"
 )
 
-func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) {
+type (
+	// Annotated is an interface for objects that have annotations
+	Annotated interface {
+		GetAnnotations() map[string]string
+	}
+	// Annotatable is an interface for objects that have annotations and `
+	Annotatable interface {
+		Annotated
+		SetAnnotations(map[string]string)
+	}
+
+	// Labeled is an interface for objects that have labels
+	Labeled interface {
+		GetLabels() map[string]string
+	}
+
+	// Labelable is an interface for objects that have labels and can set them
+	Labelable interface {
+		Labeled
+		SetLabels(map[string]string)
+	}
+)
+
+func UpdateLabels(obj Labelable, labelList []string) (bool, error) {
 	// parse strings to map
 	labels, err := parseStringMap(labelList)
 	if err != nil {
@@ -44,7 +66,7 @@ func UpdateLabels(obj metav1.Object, labelList []string) (bool, error) {
 	return changed, nil
 }
 
-func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error) {
+func UpdateAnnotations(obj Annotatable, annotationList []string) (bool, error) {
 	// parse strings to map
 	annotations, err := parseStringMap(annotationList)
 	if err != nil {
@@ -72,7 +94,7 @@ func UpdateAnnotations(obj metav1.Object, annotationList []string) (bool, error)
 
 // SetCustomLinksAnnotation sets the list of links for the UI to display next to the project member({space/virtualcluster}instance)
 // it handles unspecified links (empty) during create and update
-func SetCustomLinksAnnotation(obj metav1.Object, links []string) bool {
+func SetCustomLinksAnnotation(obj Annotatable, links []string) bool {
 	var changed bool
 	if obj == nil {
 		log.GetInstance().Error("SetCustomLinksAnnotation called on nil object")