diff --git a/test/e2e/v1alpha2/run-e2e-experiment.go b/test/e2e/v1alpha2/run-e2e-experiment.go new file mode 100644 index 00000000000..94e8b5d7dc4 --- /dev/null +++ b/test/e2e/v1alpha2/run-e2e-experiment.go @@ -0,0 +1,102 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "time" + + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "sigs.k8s.io/controller-runtime/pkg/client" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + "github.com/kubeflow/katib/pkg/util/v1alpha2/katibclient" +) + +const ( + timeout = 20 * time.Minute +) + +func verifyResult(exp *experimentsv1alpha2.Experiment) error { + if len(exp.Status.CurrentOptimalTrial.ParameterAssignments) == 0 { + return fmt.Errorf("Best parameter assignments not updated in status") + } + + if len(exp.Status.CurrentOptimalTrial.Observation.Metrics) == 0 { + return fmt.Errorf("Bst metrics not updated in status") + } + + metric := exp.Status.CurrentOptimalTrial.Observation.Metrics[0] + if metric.Name != exp.Spec.Objective.ObjectiveMetricName { + return fmt.Errorf("Best objective metric not updated in status") + } + return nil +} + +func main() { + if len(os.Args) != 2 { + log.Fatal("Experiment name is missing") + } + expName := os.Args[1] + b, err := ioutil.ReadFile(expName) + if err != nil { + log.Fatal("Error in reading file ", err) + } + exp := &experimentsv1alpha2.Experiment{} + buf := bytes.NewBufferString(string(b)) + if err = k8syaml.NewYAMLOrJSONDecoder(buf, 1024).Decode(exp); err != nil { + log.Fatal("Yaml decode error ", err) + } + kclient, err := katibclient.NewClient(client.Options{}) + if err != nil { + log.Fatal("NewClient for Katib failed: ", err) + } + var maxtrials int32 = 3 + var paralleltrials int32 = 2 + exp.Spec.MaxTrialCount = &maxtrials + exp.Spec.ParallelTrialCount = ¶lleltrials + err = kclient.CreateExperiment(exp) + if err != nil { + log.Fatal("CreateExperiment from YAML failed: ", err) + } + + for endTime := time.Now().Add(timeout); time.Now().Before(endTime); { + exp, err = kclient.GetExperiment(exp.Name, exp.Namespace) + if err != nil { + log.Fatal("Get Experiment error ", err) + } + if exp.IsCompleted() { + log.Printf("Job %v finished", exp.Name) + break + } + log.Printf("Waiting for job %v to finish. [ %v trials running %v succeeded ]", exp.Name, exp.Status.TrialsRunning, exp.Status.TrialsSucceeded) + time.Sleep(20 * time.Second) + } + + if !exp.IsCompleted() { + log.Fatal("Experiment run timed out") + } + + if exp.Status.Trials != *exp.Spec.MaxTrialCount { + log.Fatal("All trials are not run in the experiment ", exp.Status.Trials, exp.Spec.MaxTrialCount) + } + + if exp.Status.TrialsSucceeded != *exp.Spec.MaxTrialCount { + log.Fatal("All trials are not successful ", exp.Status.TrialsSucceeded, *exp.Spec.MaxTrialCount) + } + err = verifyResult(exp) + if err != nil { + log.Fatal(err) + } + + log.Printf("Experiment has recorded best current Optimal Trial %v", exp.Status.CurrentOptimalTrial) + err = kclient.DeleteExperiment(exp) + if err != nil { + log.Printf("CreateExperiment from YAML failed: %v", err) + return + } + log.Printf("Experiment %v deleted", exp.Name) +} diff --git a/test/scripts/v1alpha2/run-tests.sh b/test/scripts/v1alpha2/run-tests.sh index 2333dde03ec..2bb714882d4 100755 --- a/test/scripts/v1alpha2/run-tests.sh +++ b/test/scripts/v1alpha2/run-tests.sh @@ -66,6 +66,7 @@ echo "REPO_NAME ${REPO_NAME}" echo "VERSION ${VERSION}" sed -i -e "s@image: katib\/v1alpha2\/katib-controller@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-controller:${VERSION}@" manifests/v1alpha2/katib-controller/katib-controller.yaml +sed -i -e "s@image: katib\/v1alpha2\/metrics-collector@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/metrics-collector:${VERSION}@" manifests/v1alpha2/katib-controller/metricsControllerConfigMap.yaml sed -i -e "s@image: katib\/v1alpha2\/katib-manager@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager:${VERSION}@" manifests/v1alpha2/manager/deployment.yaml sed -i -e "s@image: katib\/v1alpha2\/katib-manager-rest@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager-rest:${VERSION}@" manifests/v1alpha2/manager-rest/deployment.yaml sed -i -e "s@image: katib\/v1alpha2\/katib-ui@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-ui:${VERSION}@" manifests/v1alpha2/ui/deployment.yaml @@ -93,6 +94,7 @@ done echo "All Katib components are running." kubectl version +kubectl cluster-info echo "Katib deployments" kubectl -n kubeflow get deploy echo "Katib services" @@ -101,7 +103,7 @@ echo "Katib pods" kubectl -n kubeflow get pod mkdir -p ${GO_DIR} -cp -r test ${GO_DIR}/test +cp -r . ${GO_DIR}/ cp -r pkg/api/v1alpha2/python/* ${GO_DIR}/test/e2e/v1alpha2 cd ${GO_DIR}/test/e2e/v1alpha2 kubectl apply -f valid-experiment.yaml @@ -126,4 +128,7 @@ python get-pip.py pip install -r test_requirements.txt python test-katib-manager.py +echo "Running e2e grid experiment" +export KUBECONFIG=$HOME/.kube/config +go run run-e2e-experiment.go ../../../examples/v1alpha2/grid-example.yaml exit 0 diff --git a/test/workflows/components/workflows-v1alpha2.libsonnet b/test/workflows/components/workflows-v1alpha2.libsonnet index 351178ec190..d18cc48f151 100644 --- a/test/workflows/components/workflows-v1alpha2.libsonnet +++ b/test/workflows/components/workflows-v1alpha2.libsonnet @@ -261,7 +261,7 @@ ], [ { - name: "run-tests", + name: "run-e2e-tests", template: "run-tests", }, ],