feat: Show all experiments in monitor (#835)

* feat: Show all experiments Signed-off-by: Ce Gao <gaoce@caicloud.io> * fix: Use namespace Signed-off-by: Ce Gao <gaoce@caicloud.io> * fix: Move package Signed-off-by: Ce Gao <gaoce@caicloud.io> * fix: Support multiple namespace Signed-off-by: Ce Gao <gaoce@caicloud.io> * feat: Support deletion Signed-off-by: Ce Gao <gaoce@caicloud.io> * feat: Support info Signed-off-by: Ce Gao <gaoce@caicloud.io> * feat: Support namesapce in submitJob Signed-off-by: Ce Gao <gaoce@caicloud.io> * fix: Remove NAS Signed-off-by: Ce Gao <gaoce@caicloud.io>
kubeflow · Sep 30, 2019 · cb6de0d · cb6de0d
1 parent 8fc9db0
commit cb6de0d
Show file tree

Hide file tree

Showing 17 changed files with 251 additions and 193 deletions.
diff --git a/cmd/ui/v1alpha3/main.go b/cmd/ui/v1alpha3/main.go
@@ -6,6 +6,8 @@ import (
 	"log"
 	"net/http"
 
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+
 	ui "github.com/kubeflow/katib/pkg/ui/v1alpha3"
 )
 
@@ -26,7 +28,7 @@ func main() {
 	frontend := http.FileServer(http.Dir(*buildDir))
 	http.Handle("/katib/", http.StripPrefix("/katib/", frontend))
 
-	http.HandleFunc("/katib/fetch_hp_jobs/", kuh.FetchHPJobs)
+	http.HandleFunc("/katib/fetch_hp_jobs/", kuh.FetchAllHPJobs)
 	http.HandleFunc("/katib/fetch_nas_jobs/", kuh.FetchNASJobs)
 	http.HandleFunc("/katib/submit_yaml/", kuh.SubmitYamlJob)
 	http.HandleFunc("/katib/submit_hp_job/", kuh.SubmitParamsJob)

diff --git a/pkg/ui/v1alpha3/backend.go b/pkg/ui/v1alpha3/backend.go
@@ -1,18 +1,16 @@
-package ui
+package v1alpha3
 
 import (
 	"context"
 	"encoding/json"
 	"log"
 	"net/http"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/ghodss/yaml"
 	"google.golang.org/grpc"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	experimentv1alpha3 "github.com/kubeflow/katib/pkg/apis/controller/experiments/v1alpha3"
@@ -46,29 +44,11 @@ func (k *KatibUIHandler) connectManager() (*grpc.ClientConn, api_pb_v1alpha3.Man
 
 func (k *KatibUIHandler) FetchHPJobs(w http.ResponseWriter, r *http.Request) {
 	//enableCors(&w)
-
-	jobs := make([]JobView, 0)
-
-	el, err := k.katibClient.GetExperimentList()
+	jobs, err := k.getExperimentList(consts.DefaultKatibNamespace, JobTypeHP)
 	if err != nil {
-		log.Printf("GetExperimentList for HP failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	for _, experiment := range el.Items {
-		if experiment.Spec.Parameters != nil {
-			experimentLastCondition, err := experiment.GetLastConditionType()
-			if err != nil {
-				log.Printf("GetLastConditionType for HP failed: %v", err)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-			jobs = append(jobs, JobView{
-				Name:   experiment.Name,
-				Status: string(experimentLastCondition),
-			})
-		}
-	}
 
 	response, err := json.Marshal(jobs)
 	if err != nil {
@@ -80,40 +60,21 @@ func (k *KatibUIHandler) FetchHPJobs(w http.ResponseWriter, r *http.Request) {
 
 }
 
-func (k *KatibUIHandler) FetchNASJobs(w http.ResponseWriter, r *http.Request) {
-	//enableCors(&w)
-
-	jobs := make([]JobView, 0)
-
-	el, err := k.katibClient.GetExperimentList()
+// FetchAllHPJobs gets experiments in all namespaces.
+func (k *KatibUIHandler) FetchAllHPJobs(w http.ResponseWriter, r *http.Request) {
+	// Use "" to get experiments in all namespaces.
+	jobs, err := k.getExperimentList("", JobTypeHP)
 	if err != nil {
-		log.Printf("GetExperimentList for NAS failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	for _, experiment := range el.Items {
-		if experiment.Spec.NasConfig != nil {
-			experimentLastCondition, err := experiment.GetLastConditionType()
-			if err != nil {
-				log.Printf("GetLastConditionType for HP failed: %v", err)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-			jobs = append(jobs, JobView{
-				Name:   experiment.Name,
-				Status: string(experimentLastCondition),
-			})
-		}
-	}
-
 	response, err := json.Marshal(jobs)
 	if err != nil {
-		log.Printf("Marshal NAS jobs failed: %v", err)
+		log.Printf("Marshal HP jobs failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	w.Write(response)
-
 }
 
 func (k *KatibUIHandler) SubmitYamlJob(w http.ResponseWriter, r *http.Request) {
@@ -178,7 +139,9 @@ func (k *KatibUIHandler) SubmitParamsJob(w http.ResponseWriter, r *http.Request)
 
 func (k *KatibUIHandler) DeleteExperiment(w http.ResponseWriter, r *http.Request) {
 	experimentName := r.URL.Query()["experimentName"][0]
-	experiment, err := k.katibClient.GetExperiment(experimentName)
+	namespace := r.URL.Query()["namespace"][0]
+
+	experiment, err := k.katibClient.GetExperiment(experimentName, namespace)
 	if err != nil {
 		log.Printf("GetExperiment failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -195,12 +158,13 @@ func (k *KatibUIHandler) DeleteExperiment(w http.ResponseWriter, r *http.Request
 func (k *KatibUIHandler) FetchHPJobInfo(w http.ResponseWriter, r *http.Request) {
 	//enableCors(&w)
 	experimentName := r.URL.Query()["experimentName"][0]
+	namespace := r.URL.Query()["namespace"][0]
 
 	conn, c := k.connectManager()
 	defer conn.Close()
 
 	resultText := "trialName"
-	experiment, err := k.katibClient.GetExperiment(experimentName)
+	experiment, err := k.katibClient.GetExperiment(experimentName, namespace)
 	if err != nil {
 		log.Printf("GetExperiment from HP job failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -272,7 +236,6 @@ func (k *KatibUIHandler) FetchHPJobInfo(w http.ResponseWriter, r *http.Request)
 		return
 	}
 	w.Write(response)
-
 }
 
 func (k *KatibUIHandler) FetchHPJobTrialInfo(w http.ResponseWriter, r *http.Request) {
@@ -314,82 +277,6 @@ func (k *KatibUIHandler) FetchHPJobTrialInfo(w http.ResponseWriter, r *http.Requ
 	w.Write(response)
 }
 
-func (k *KatibUIHandler) FetchNASJobInfo(w http.ResponseWriter, r *http.Request) {
-	//enableCors(&w)
-	experimentName := r.URL.Query()["experimentName"][0]
-
-	responseRaw := make([]NNView, 0)
-	var architecture string
-	var decoder string
-
-	conn, c := k.connectManager()
-
-	defer conn.Close()
-
-	trials, err := k.katibClient.GetTrialList(experimentName)
-	if err != nil {
-		log.Printf("GetTrialList from NAS job failed: %v", err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	log.Printf("Got Trial List")
-
-	for i, t := range trials.Items {
-		succeeded := false
-		for _, condition := range t.Status.Conditions {
-			if condition.Type == trialsv1alpha3.TrialSucceeded {
-				succeeded = true
-			}
-		}
-		if succeeded {
-			obsLogResp, err := c.GetObservationLog(
-				context.Background(),
-				&api_pb_v1alpha3.GetObservationLogRequest{
-					TrialName: t.Name,
-					StartTime: "",
-					EndTime:   "",
-				},
-			)
-			if err != nil {
-				log.Printf("GetObservationLog from NAS job failed: %v", err)
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-			metricsName := make([]string, 0)
-			metricsValue := make([]string, 0)
-			for _, m := range obsLogResp.ObservationLog.MetricLogs {
-				metricsName = append(metricsName, m.Metric.Name)
-				metricsValue = append(metricsValue, m.Metric.Value)
-
-			}
-			for _, trialParam := range t.Spec.ParameterAssignments {
-				if trialParam.Name == "architecture" {
-					architecture = trialParam.Value
-				}
-				if trialParam.Name == "nn_config" {
-					decoder = trialParam.Value
-				}
-			}
-			responseRaw = append(responseRaw, NNView{
-				Name:         "Generation " + strconv.Itoa(i),
-				TrialName:    t.Name,
-				Architecture: generateNNImage(architecture, decoder),
-				MetricsName:  metricsName,
-				MetricsValue: metricsValue,
-			})
-		}
-	}
-	log.Printf("Logs parsed, result: %v", responseRaw)
-
-	response, err := json.Marshal(responseRaw)
-	if err != nil {
-		log.Printf("Marshal result in NAS job failed: %v", err)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	w.Write(response)
-}
-
 // FetchTrialTemplates gets the trial templates for the given namespace.
 func (k *KatibUIHandler) FetchTrialTemplates(w http.ResponseWriter, r *http.Request) {
 	//enableCors(&w)

diff --git a/pkg/ui/v1alpha3/frontend/src/actions/generalActions.js b/pkg/ui/v1alpha3/frontend/src/actions/generalActions.js
@@ -28,16 +28,18 @@ export const DELETE_EXPERIMENT_REQUEST = "DELETE_EXPERIMENT_REQUEST";
 export const DELETE_EXPERIMENT_FAILURE = "DELETE_EXPERIMENT_FAILURE";
 export const DELETE_EXPERIMENT_SUCCESS = "DELETE_EXPERIMENT_SUCCESS";
 
-export const deleteExperiment = (experimentName) => ({
+export const deleteExperiment = (name, namespace) => ({
     type: DELETE_EXPERIMENT_REQUEST,
-    experimentName,
+    name,
+    namespace,
 })
 
 export const OPEN_DELETE_EXPERIMENT_DIALOG = "OPEN_DELETE_EXPERIMENT_DIALOG";
 
-export const openDeleteExperimentDialog = (experimentName) => ({
+export const openDeleteExperimentDialog = (name, namespace) => ({
     type: OPEN_DELETE_EXPERIMENT_DIALOG,
-    experimentName,
+    name,
+    namespace,
 })
 
 export const CLOSE_DELETE_EXPERIMENT_DIALOG = "CLOSE_DELETE_EXPERIMENT_DIALOG";

diff --git a/pkg/ui/v1alpha3/frontend/src/actions/hpCreateActions.js b/pkg/ui/v1alpha3/frontend/src/actions/hpCreateActions.js
@@ -121,6 +121,13 @@ export const changeTrial = (trial) => ({
     trial,
 })
 
+export const CHANGE_TRIAL_NAMESPACE_HP = "CHANGE_TRIAL_NAMESPACE_HP";
+
+export const changeTrialNamespace = (namespace) => ({
+    type: CHANGE_TRIAL_NAMESPACE_HP,
+    namespace,
+})
+
 export const SUBMIT_HP_JOB_REQUEST = "SUBMIT_HP_JOB_REQUEST";
 export const SUBMIT_HP_JOB_SUCCESS = "SUBMIT_HP_JOB_SUCCESS";
 export const SUBMIT_HP_JOB_FAILURE = "SUBMIT_HP_JOB_FAILURE";

diff --git a/pkg/ui/v1alpha3/frontend/src/actions/hpMonitorActions.js b/pkg/ui/v1alpha3/frontend/src/actions/hpMonitorActions.js
@@ -24,9 +24,10 @@ export const FETCH_HP_JOB_INFO_REQUEST = "FETCH_HP_JOB_INFO_REQUEST";
 export const FETCH_HP_JOB_INFO_SUCCESS = "FETCH_HP_JOB_INFO_SUCCESS";
 export const FETCH_HP_JOB_INFO_FAILURE = "FETCH_HP_JOB_INFO_FAILURE";
 
-export const fetchHPJobInfo = (experimentName) => ({
+export const fetchHPJobInfo = (name, namespace) => ({
     type: FETCH_HP_JOB_INFO_REQUEST,
-    experimentName
+    name,
+    namespace,
 })
 
 export const FETCH_HP_JOB_TRIAL_INFO_REQUEST = "FETCH_HP_JOB_TRIAL_INFO_REQUEST";

diff --git a/pkg/ui/v1alpha3/frontend/src/components/App.jsx b/pkg/ui/v1alpha3/frontend/src/components/App.jsx
@@ -32,7 +32,7 @@ const App = (props) => {
             <Route exact path="/" component={Main} />
             <Route path="/katib/hp" component={HP} />
             <Route exact path="/katib/hp_monitor" component={HPJobMonitor} />
-            <Route path="/katib/hp_monitor/:name" component={HPJobInfo} />
+            <Route path="/katib/hp_monitor/:namespace/:name" component={HPJobInfo} />
             <Route path="/katib/nas" component={NAS} />
             <Route exact path="/katib/nas_monitor" component={NASJobMonitor} />
             <Route path="/katib/nas_monitor/:name" component={NASJobInfo} />

diff --git a/pkg/ui/v1alpha3/frontend/src/components/HP/Create/HPParameters.jsx b/pkg/ui/v1alpha3/frontend/src/components/HP/Create/HPParameters.jsx
@@ -175,7 +175,8 @@ const mapStateToProps = (state) => ({
     algorithmName: state[module].algorithmName,
     algorithmSettings: state[module].algorithmSettings,
     parameters: state[module].parameters,
-    trial: state[module].trial
+    trial: state[module].trial,
+    trialNamespace: state[module].trialNamespace,
 })
 
 //TODO: Added validation and remove it

diff --git a/pkg/ui/v1alpha3/frontend/src/components/HP/Create/Params/Trial.jsx b/pkg/ui/v1alpha3/frontend/src/components/HP/Create/Params/Trial.jsx
@@ -12,7 +12,7 @@ import Select from '@material-ui/core/Select';
 import TextField from '@material-ui/core/TextField';
 
 import { connect } from 'react-redux';
-import { changeTrial } from '../../../../actions/hpCreateActions';
+import { changeTrial, changeTrialNamespace } from '../../../../actions/hpCreateActions';
 import { fetchTrialTemplates } from '../../../../actions/templateActions';
 
 const module = "hpCreate";
@@ -44,6 +44,7 @@ class TrialSpecParam extends React.Component {
 
     onTrialNamespaceChange = (event) => {
         this.props.fetchTrialTemplates(event.target.value);
+        this.props.changeTrialNamespace(event.target.value);
     }
 
     onTrialChange = (event) => {
@@ -118,7 +119,8 @@ const mapStateToProps = state => {
     return {
         trial: state[module].trial,
         templates: state[templateModule].trialTemplates,
+        trialNamespace: state[module].trialNamespace,
     }
 }
 
-export default connect(mapStateToProps, { changeTrial, fetchTrialTemplates })(withStyles(styles)(TrialSpecParam));
+export default connect(mapStateToProps, { changeTrialNamespace, changeTrial, fetchTrialTemplates })(withStyles(styles)(TrialSpecParam));
diff --git a/pkg/ui/v1alpha3/frontend/src/components/HP/Monitor/HPJobInfo.jsx b/pkg/ui/v1alpha3/frontend/src/components/HP/Monitor/HPJobInfo.jsx
@@ -33,7 +33,8 @@ const styles = theme => ({
 class HPJobInfo extends React.Component {
 
     componentDidMount() {
-        this.props.fetchHPJobInfo(this.props.match.params.name);
+        this.props.fetchHPJobInfo(
+            this.props.match.params.name, this.props.match.params.namespace);
     }
 
     render () {
@@ -52,6 +53,9 @@ class HPJobInfo extends React.Component {
                     <Typography  className = {classes.header} variant={"h5"}>
                         Experiment Name: {this.props.match.params.name}
                     </Typography>
+                    <Typography  className = {classes.header} variant={"h5"}>
+                        Experiment Namespace: {this.props.match.params.namespace}
+                    </Typography>
                     <br />
                     <HPJobPlot name={this.props.match.params.name} />
                     <HPJobTable name={this.props.match.params.name} />

diff --git a/pkg/ui/v1alpha3/frontend/src/components/HP/Monitor/HPJobList.jsx b/pkg/ui/v1alpha3/frontend/src/components/HP/Monitor/HPJobList.jsx
@@ -43,8 +43,8 @@ const HPJobList = (props) => {
 
     const { classes } = props;
 
-    const onDeleteExperiment = (experimentName) => (event) => {
-        props.openDeleteExperimentDialog(experimentName);
+    const onDeleteExperiment = (name, namespace) => (event) => {
+        props.openDeleteExperimentDialog(name, namespace);
     }
 
     return (
@@ -64,13 +64,13 @@ const HPJobList = (props) => {
                         icon = (<HighlightOffIcon className={classes.failed}/>)
                     }
                     return (
-                        <ListItem button key={i} component={Link} to={`/katib/hp_monitor/${job.name}`}>
+                        <ListItem button key={i} component={Link} to={`/katib/hp_monitor/${job.namespace}/${job.name}`}>
                             <ListItemIcon>
                                 {icon}
                             </ListItemIcon>
-                            <ListItemText inset primary={job.name} />
+                            <ListItemText inset primary={`${job.name}`} secondary={job.namespace} />
                             <ListItemSecondaryAction>
-                                <IconButton aria-label={"Delete"} onClick={onDeleteExperiment(job.name)}>
+                                <IconButton aria-label={"Delete"} onClick={onDeleteExperiment(job.name, job.namespace)}>
                                     <DeleteIcon />
                                 </IconButton>
                             </ListItemSecondaryAction>

diff --git a/pkg/ui/v1alpha3/frontend/src/components/Menu/DeleteDialog.jsx b/pkg/ui/v1alpha3/frontend/src/components/Menu/DeleteDialog.jsx
@@ -22,7 +22,7 @@ const DeleteDialog = (props) => {
     const classes = useStyles();
 
     const onDelete = () => {
-        props.deleteExperiment(props.deleteExperimentName);
+        props.deleteExperiment(props.deleteExperimentName, props.deleteExperimentNamespace);
     }
 
     return (
@@ -53,6 +53,7 @@ const DeleteDialog = (props) => {
 const mapStateToProps = (state) => ({
     open: state[module].deleteDialog,
     deleteExperimentName: state[module].deleteExperimentName,
+    deleteExperimentNamespace: state[module].deleteExperimentNamespace,
 })
 
 export default connect(mapStateToProps, { closeDeleteExperimentDialog, deleteExperiment })(DeleteDialog);