Resume experiment with extra trials from last checkpoint

pkg/apis/controller/experiments/v1alpha3/util.go

@@ -93,6 +93,14 @@ func (exp *Experiment) IsCompleted() bool {
 	return exp.IsSucceeded() || exp.IsFailed()
 }
 
+func (exp *Experiment) IsCompletedReason(reason string) bool {
+	cond := getCondition(exp, ExperimentSucceeded)
+	if cond != nil && cond.Status == v1.ConditionTrue && cond.Reason == reason {
+		return true
+	}
+	return false
+}
+
 func (exp *Experiment) HasRunningTrials() bool {
 	return exp.Status.TrialsRunning != 0
 }
@@ -131,6 +139,12 @@ func (exp *Experiment) MarkExperimentStatusRunning(reason, message string) {
 	exp.setCondition(ExperimentRunning, v1.ConditionTrue, reason, message)
 }
 
+func (exp *Experiment) MarkExperimentStatusRestarting(reason, message string) {
+	exp.removeCondition(ExperimentSucceeded)
+	exp.removeCondition(ExperimentFailed)
+	exp.setCondition(ExperimentRestarting, v1.ConditionTrue, reason, message)
+}
+
 func (exp *Experiment) MarkExperimentStatusSucceeded(reason, message string) {
 	currentCond := getCondition(exp, ExperimentRunning)
 	if currentCond != nil {

pkg/controller.v1alpha3/experiment/experiment_controller.go

@@ -186,8 +186,22 @@ func (r *ReconcileExperiment) Reconcile(request reconcile.Request) (reconcile.Re
 		return r.updateFinalizers(instance, finalizers)
 	}
 
-	if instance.IsCompleted() && !instance.HasRunningTrials() {
-		return reconcile.Result{}, nil
+	if instance.IsCompleted() {
+		// Check if completed instance is restartable
+		// Experiment is restartable only if it is in succeeded state by reaching max trials
+		if util.IsCompletedExperimentRestartable(instance) {
+			// Check if max trials is reconfigured
+			if (instance.Spec.MaxTrialCount != nil) &&
+				(*instance.Spec.MaxTrialCount != instance.Status.Trials) {
+				msg := "Experiment is restarted"
+				instance.MarkExperimentStatusRestarting(util.ExperimentRestartingReason, msg)
+			}
+		} else {
+			// If experiment is completed with no running trials, stop reconcile
+			if !instance.HasRunningTrials() {
+				return reconcile.Result{}, nil
+			}
+		}
 	}
 	if !instance.IsCreated() {
 		if instance.Status.StartTime == nil {

pkg/controller.v1alpha3/experiment/util/status_util.go

@@ -27,11 +27,14 @@ import (
 var log = logf.Log.WithName("experiment-status-util")
 
 const (
-	ExperimentCreatedReason   = "ExperimentCreated"
-	ExperimentRunningReason   = "ExperimentRunning"
-	ExperimentSucceededReason = "ExperimentSucceeded"
-	ExperimentFailedReason    = "ExperimentFailed"
-	ExperimentKilledReason    = "ExperimentKilled"
+	ExperimentCreatedReason              = "ExperimentCreated"
+	ExperimentRunningReason              = "ExperimentRunning"
+	ExperimentRestartingReason           = "ExperimentRestarting"
+	ExperimentGoalReachedReason          = "ExperimentGoalReached"
+	ExperimentMaxTrialsReachedReason     = "ExperimentMaxTrialsReached"
+	ExperimentSuggestionEndReachedReason = "ExperimentSuggestionEndReached"
+	ExperimentFailedReason               = "ExperimentFailed"
+	ExperimentKilledReason               = "ExperimentKilled"
 )
 
 func UpdateExperimentStatus(instance *experimentsv1alpha3.Experiment, trials *trialsv1alpha3.TrialList) error {
@@ -143,23 +146,23 @@ func UpdateExperimentStatusCondition(instance *experimentsv1alpha3.Experiment, i
 
 	if isObjectiveGoalReached {
 		msg := "Experiment has succeeded because Objective goal has reached"
-		instance.MarkExperimentStatusSucceeded(ExperimentSucceededReason, msg)
+		instance.MarkExperimentStatusSucceeded(ExperimentGoalReachedReason, msg)
 		instance.Status.CompletionTime = &now
 		IncreaseExperimentsSucceededCount()
 		return
 	}
 
 	if (instance.Spec.MaxTrialCount != nil) && (completedTrialsCount >= *instance.Spec.MaxTrialCount) {
 		msg := "Experiment has succeeded because max trial count has reached"
-		instance.MarkExperimentStatusSucceeded(ExperimentSucceededReason, msg)
+		instance.MarkExperimentStatusSucceeded(ExperimentMaxTrialsReachedReason, msg)
 		instance.Status.CompletionTime = &now
 		IncreaseExperimentsSucceededCount()
 		return
 	}
 
 	if getSuggestionDone && (instance.Status.TrialsPending+instance.Status.TrialsRunning) == 0 {
 		msg := "Experiment has succeeded because suggestion service has reached the end"
-		instance.MarkExperimentStatusSucceeded(ExperimentSucceededReason, msg)
+		instance.MarkExperimentStatusSucceeded(ExperimentSuggestionEndReachedReason, msg)
 		instance.Status.CompletionTime = &now
 		IncreaseExperimentsSucceededCount()
 		return
@@ -176,3 +179,10 @@ func UpdateExperimentStatusCondition(instance *experimentsv1alpha3.Experiment, i
 	msg := "Experiment is running"
 	instance.MarkExperimentStatusRunning(ExperimentRunningReason, msg)
 }
+
+func IsCompletedExperimentRestartable(instance *experimentsv1alpha3.Experiment) bool {
+	if instance.IsSucceeded() && instance.IsCompletedReason(ExperimentMaxTrialsReachedReason) {
+		return true
+	}
+	return false
+}