From fcd100561ae033a4a1de08191aa84f0f33e44758 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Sun, 1 Jul 2018 15:06:17 -0700 Subject: [PATCH] Move the GKEDemo into kubeflow/examples (#135) * The GKEDemo is using the GitHub summarization example; I think we should put all of the code for that demo kubeflow/examples (see kubeflow/examples#161) * The main code is the Katib HP controller git-issue-summarize-demo.go * We don't need the manifests for deploying katib because we can deploy Katib using the Kubeflow ksonnet package. * The code in docker-image duplicates the code in kubeflow/examples so we shouldn't need it. Related to: #116 --- examples/GKEDemo/GKE-Demo.md | 160 ------- examples/GKEDemo/deploy.sh | 19 - examples/GKEDemo/destroy.sh | 5 - examples/GKEDemo/docker-image/Dockerfile | 7 - examples/GKEDemo/docker-image/Dockerfile-gpu | 7 - .../docker-image/IssueSummarization.py | 25 - .../GKEDemo/docker-image/requirements.txt | 11 - .../GKEDemo/docker-image/seq2seq_utils.py | 432 ------------------ examples/GKEDemo/docker-image/train.py | 211 --------- examples/GKEDemo/down.sh | 3 - examples/GKEDemo/git-issue-summarize-demo.go | 208 --------- examples/GKEDemo/manifests/0-namespace.yaml | 4 - .../manifests/modeldb/backend/deployment.yaml | 32 -- .../manifests/modeldb/backend/service.yaml | 17 - .../manifests/modeldb/db/deployment.yaml | 30 -- .../GKEDemo/manifests/modeldb/db/service.yaml | 17 - .../modeldb/frontend/deployment.yaml | 36 -- .../manifests/modeldb/frontend/ingress.yaml | 9 - .../manifests/modeldb/frontend/service.yaml | 17 - .../manifests/vizier/core/deployment.yaml | 37 -- .../manifests/vizier/core/ingress.yaml | 9 - .../GKEDemo/manifests/vizier/core/rbac.yaml | 37 -- .../manifests/vizier/core/service.yaml | 17 - .../manifests/vizier/db/deployment.yaml | 37 -- .../GKEDemo/manifests/vizier/db/service.yaml | 17 - .../medianstopping/deployment.yaml | 30 -- .../earlystopping/medianstopping/service.yaml | 17 - .../vizier/suggestion/grid/deployment.yaml | 30 -- .../vizier/suggestion/grid/service.yaml | 17 - .../vizier/suggestion/random/deployment.yaml | 30 -- .../vizier/suggestion/random/service.yaml | 17 - 31 files changed, 1545 deletions(-) delete mode 100644 examples/GKEDemo/GKE-Demo.md delete mode 100755 examples/GKEDemo/deploy.sh delete mode 100755 examples/GKEDemo/destroy.sh delete mode 100644 examples/GKEDemo/docker-image/Dockerfile delete mode 100644 examples/GKEDemo/docker-image/Dockerfile-gpu delete mode 100644 examples/GKEDemo/docker-image/IssueSummarization.py delete mode 100644 examples/GKEDemo/docker-image/requirements.txt delete mode 100644 examples/GKEDemo/docker-image/seq2seq_utils.py delete mode 100644 examples/GKEDemo/docker-image/train.py delete mode 100755 examples/GKEDemo/down.sh delete mode 100644 examples/GKEDemo/git-issue-summarize-demo.go delete mode 100644 examples/GKEDemo/manifests/0-namespace.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/backend/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/backend/service.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/db/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/db/service.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/frontend/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/frontend/ingress.yaml delete mode 100644 examples/GKEDemo/manifests/modeldb/frontend/service.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/core/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/core/ingress.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/core/rbac.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/core/service.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/db/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/db/service.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/service.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/suggestion/grid/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/suggestion/grid/service.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/suggestion/random/deployment.yaml delete mode 100644 examples/GKEDemo/manifests/vizier/suggestion/random/service.yaml diff --git a/examples/GKEDemo/GKE-Demo.md b/examples/GKEDemo/GKE-Demo.md deleted file mode 100644 index 269afa43c0b..00000000000 --- a/examples/GKEDemo/GKE-Demo.md +++ /dev/null @@ -1,160 +0,0 @@ -# Simple GKE Demo -You can deploy katib components and try simple mnist demo on the cloud! - -## build your training docker image -The sample code is docker-image directory. -It is based on [KubeFlow's Github issue summaraize example](https://github.com/kubeflow/examples/tree/master/github_issue_summarization/notebooks) image. - -## deploy Cluster -This is grid parameter search demo for [KubeFlow's Github issue summaraize example](https://github.com/kubeflow/examples/tree/master/github_issue_summarization) - -Let's deploy GKE cluster by gcloud commnad. -``` -gcloud container clusters create --machine-type n1-highmem-4 --num-nodes 2 katib -``` - - -Then deploy Katib compenents. - -``` -kubectl config set-credentials temp-admin --username=admin --password=$(gcloud container clusters describe katib --format="value(masterAuth.password)") -kubectl config set-context temp-context --cluster=$(kubectl config get-clusters | grep katib) --user=temp-admin -kubectl config use-context temp-context -kubectl apply -f manifests/0-namespace.yaml -kubectl --namespace=${NAMESPACE} create secret generic gcp-credentials --from-file=key.json="${KEY_FILE}" -kubectl apply -f manifests/modeldb/db -kubectl apply -f manifests/modeldb/backend -kubectl apply -f manifests/modeldb/frontend -kubectl apply -f manifests/vizier/db -kubectl apply -f manifests/vizier/core -kubectl apply -f manifests/vizier/suggestion/random -kubectl apply -f manifests/vizier/suggestion/grid -kubectl apply -f manifests/vizier/earlystopping/medianstopping -``` - -In this demo, katib components does not export the port of services. -You need to start port-forward for katib services `6789 -> manager` and `3000 -> UI`. - -``` -$ kubectl -n katib port-forward svc/vizier-core 6789:6789 & -$ kubectl -n katib port-forward svc/modeldb-frontend 3000:3000 & -``` - -If you don't want to port forward, you should set firewall to allow the ports or set up Ingress. - -### Push data to GCS -If you want to pull input data and push logs to google cloud storage, You need to confidurate GCP service account. - -See also https://github.com/kubeflow/examples/blob/master/github_issue_summarization/training_the_model_tfjob.md - -## Create Study -Run study with `go run`. you should set endpoint of vizier-core. - -``` -go run git-issue-summarize-demo.go -s {{ip-addr of node vizier-core deployed}}:30678 -``` -Katib will make 2 girds of learling-rate parameter from 0.005 to 0.5. - -The `git-issue-summarize-demo.go` is a controller of this study. -It sets hyper parameter config, suggestion config, k8s job config and then, create study and run trial with katib-API. - -The training logic is `docker-image/train.py` - -## UI -You can check your Model with Web UI. - -Acsess to `http://127.0.0.1:3000/` - -The Results will be saved automatically. - -## Description of git-issue-summarize-demo.go -You can make hyperparameter and evaluate it by Katib-API. -Katib-APIs are grpc. So you can use any language grpc supported(e.g. golang, python, c++). -A typical case, you will call APIs in the order as below. -In git-issue-summarize-demo.go, it wait for the status of all workers will be Completed. - -### CreateStudy -First, you should create Study. -The input is StudyConfig. -It has Study name, owner, optimization info, and Parameter config(parameter name, min, and max). -This function generates a unique ID for your study and stores the config to DB. -Input: -* StudyConfig: - * Name: string - * Owner: string - * OptimizationType: enum(OptimizationType_MAXIMIZE, OptimizationType_MINIMIZE) - * OptimizationGoal: float - * DefaultSuggestionAlgorithm: string - * DefaultEarlyStoppingAlgorithm: string - * ObjectiveValueName: string - * Metrics: List of Metrics name - * ParameterConfigs: List of parameter config. -Return: -* StudyID - -### SetSuggestionParameters -Hyperparameters are generated by suggestion services with Parameter config of Study. -You can set the specific config for each suggestion. -Input: -* StudyID: ID of your study. -* SuggestionAlgorithm: name of suggestion service (e.g. random, grid) -* SuggestionParameters: key-value pairs parameter for suggestions. The wanted key is different for each suggestion. -Return: -* ParameterID - -### GetSuggestions -This function will create Trials(set of Parameters). -Input: -* StudyID: ID of your study. -* SuggestionAlgorithm: name of suggestion service (e.g. random, grid) -* RequestNumber: the number you want to evaluate. -* ParamID: ParameterID you got from SetSuggestionParameters func. -Return -* List of Trials - * TrialID - * Parameter Sets - -### RunTrial -Start to evaluate Trial. -When you use kubernetes runtime, the pods are created the specified config. -Input: -* StudyId: ID of your study. -* TrialId: ID of Trial. -* Runtime: worker type(e.g. kubernetes) -* WorkerConfig: runtime config - * Image: name of docker image - * Command: running commands - * GPU: number of GPU - * Scheduler: scheduler name -Return: -* List of WorkerID - -### GetMetrics -Get metrics of running workers. -Input: -* StudyId: ID of your study. -* WorkerIDs: List of worker ID you want to get metrics from. -Return: -* List of Metrics - -### SaveModel -Save the Model date to KatibDB. After you called this function, you can look model info in the KatibUI. -When you call this API multiple time, only Metrics will be updated. -Input: -* ModelInfo - * StudyName - * WorkerId - * Parameters: List of Parameter - * Metrics: List of Metrics - * ModelPath: path to model saved. (PVCname:mountpath) -* DataSet: informatino of input date - * Name - * Path: path to input data.(PVCname:mountpath) - -Return: - -### GetWorkers -You can get worker list and status of workers. -Input: -Return: -* List of worker information diff --git a/examples/GKEDemo/deploy.sh b/examples/GKEDemo/deploy.sh deleted file mode 100755 index d5302b52b1c..00000000000 --- a/examples/GKEDemo/deploy.sh +++ /dev/null @@ -1,19 +0,0 @@ -#/bin/bash -set -x -gcloud container clusters create --machine-type n1-highmem-4 --num-nodes 2 katib -SERVICE_ACCOUNT=github-issue-summarization -set -e -kubectl config set-credentials temp-admin --username=admin --password=$(gcloud container clusters describe katib --format="value(masterAuth.password)") -kubectl config set-context temp-context --cluster=$(kubectl config get-clusters | grep katib) --user=temp-admin -kubectl config use-context temp-context -kubectl apply -f manifests/0-namespace.yaml -kubectl apply -f manifests/modeldb/db -kubectl apply -f manifests/modeldb/backend -kubectl apply -f manifests/modeldb/frontend -kubectl apply -f manifests/vizier/db -kubectl apply -f manifests/vizier/core -kubectl apply -f manifests/vizier/suggestion/random -kubectl apply -f manifests/vizier/suggestion/grid -kubectl apply -f manifests/vizier/earlystopping/medianstopping -#gcloud compute firewall-rules create katibservice --allow tcp:30080,tcp:30678 -gcloud compute instances list diff --git a/examples/GKEDemo/destroy.sh b/examples/GKEDemo/destroy.sh deleted file mode 100755 index ed99489cf13..00000000000 --- a/examples/GKEDemo/destroy.sh +++ /dev/null @@ -1,5 +0,0 @@ -#/bin/bash -set -x -#gcloud compute firewall-rules delete katibservice -gcloud container clusters delete katib -pkill kubectl diff --git a/examples/GKEDemo/docker-image/Dockerfile b/examples/GKEDemo/docker-image/Dockerfile deleted file mode 100644 index 9c3bf2ef03c..00000000000 --- a/examples/GKEDemo/docker-image/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM gcr.io/kubeflow-images-staging/tensorflow-1.6.0-notebook-cpu -COPY requirements.txt /workdir/requirements.txt -RUN pip install -r /workdir/requirements.txt -RUN pip install google.cloud sklearn ktext annoy nltk -COPY train.py /workdir/train.py -COPY seq2seq_utils.py /workdir/seq2seq_utils.py -COPY IssueSummarization.py /workdir/IssueSummarization.py diff --git a/examples/GKEDemo/docker-image/Dockerfile-gpu b/examples/GKEDemo/docker-image/Dockerfile-gpu deleted file mode 100644 index dc7e88e9411..00000000000 --- a/examples/GKEDemo/docker-image/Dockerfile-gpu +++ /dev/null @@ -1,7 +0,0 @@ -FROM gcr.io/kubeflow-images-staging/tensorflow-1.6.0-notebook-gpu -COPY requirements.txt /workdir/requirements.txt -RUN pip install -r /workdir/requirements.txt -RUN pip install google.cloud sklearn ktext annoy nltk -COPY train.py /workdir/train.py -COPY seq2seq_utils.py /workdir/seq2seq_utils.py -COPY IssueSummarization.py /workdir/IssueSummarization.py diff --git a/examples/GKEDemo/docker-image/IssueSummarization.py b/examples/GKEDemo/docker-image/IssueSummarization.py deleted file mode 100644 index b992b2ddc3a..00000000000 --- a/examples/GKEDemo/docker-image/IssueSummarization.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Generates predictions using a stored model. - -Uses trained model files to generate a prediction. -""" - -from __future__ import print_function - -import numpy as np -import dill as dpickle -from keras.models import load_model -from seq2seq_utils import Seq2Seq_Inference - -class IssueSummarization(object): - - def __init__(self): - with open('body_pp.dpkl', 'rb') as body_file: - body_pp = dpickle.load(body_file) - with open('title_pp.dpkl', 'rb') as title_file: - title_pp = dpickle.load(title_file) - self.model = Seq2Seq_Inference(encoder_preprocessor=body_pp, - decoder_preprocessor=title_pp, - seq2seq_model=load_model('seq2seq_model_tutorial.h5')) - - def predict(self, input_text, feature_names): # pylint: disable=unused-argument - return np.asarray([[self.model.generate_issue_title(body[0])[1]] for body in input_text]) diff --git a/examples/GKEDemo/docker-image/requirements.txt b/examples/GKEDemo/docker-image/requirements.txt deleted file mode 100644 index 27f1c6e8992..00000000000 --- a/examples/GKEDemo/docker-image/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -numpy -keras -dill -matplotlib -tensorflow -annoy -tqdm -nltk -IPython -ktext -h5py diff --git a/examples/GKEDemo/docker-image/seq2seq_utils.py b/examples/GKEDemo/docker-image/seq2seq_utils.py deleted file mode 100644 index 0ddaebfabe1..00000000000 --- a/examples/GKEDemo/docker-image/seq2seq_utils.py +++ /dev/null @@ -1,432 +0,0 @@ -import logging -import dill as dpickle -import numpy as np -from matplotlib import pyplot as plt -import tensorflow as tf -from IPython.display import SVG, display -from keras import backend as K -from keras.layers import Input -from keras.models import Model -from keras.utils.vis_utils import model_to_dot -from annoy import AnnoyIndex -from tqdm import tqdm, tqdm_notebook -from nltk.translate.bleu_score import corpus_bleu - - -def load_text_processor(fname='title_pp.dpkl'): - """ - Load preprocessors from disk. - - Parameters - ---------- - fname: str - file name of ktext.proccessor object - - Returns - ------- - num_tokens : int - size of vocabulary loaded into ktext.processor - pp : ktext.processor - the processor you are trying to load - - Typical Usage: - ------------- - - num_decoder_tokens, title_pp = load_text_processor(fname='title_pp.dpkl') - num_encoder_tokens, body_pp = load_text_processor(fname='body_pp.dpkl') - - """ - # Load files from disk - with open(fname, 'rb') as f: - pp = dpickle.load(f) - - num_tokens = max(pp.id2token.keys()) + 1 - print('Size of vocabulary for {}: {}'.format(fname, num_tokens)) - return num_tokens, pp - - -def load_decoder_inputs(decoder_np_vecs='train_title_vecs.npy'): - """ - Load decoder inputs. - - Parameters - ---------- - decoder_np_vecs : str - filename of serialized numpy.array of decoder input (issue title) - - Returns - ------- - decoder_input_data : numpy.array - The data fed to the decoder as input during training for teacher forcing. - This is the same as `decoder_np_vecs` except the last position. - decoder_target_data : numpy.array - The data that the decoder data is trained to generate (issue title). - Calculated by sliding `decoder_np_vecs` one position forward. - - """ - vectorized_title = np.load(decoder_np_vecs) - # For Decoder Input, you don't need the last word as that is only for prediction - # when we are training using Teacher Forcing. - decoder_input_data = vectorized_title[:, :-1] - - # Decoder Target Data Is Ahead By 1 Time Step From Decoder Input Data (Teacher Forcing) - decoder_target_data = vectorized_title[:, 1:] - - print('Shape of decoder input: {}'.format(decoder_input_data.shape)) - print('Shape of decoder target: {}'.format(decoder_target_data.shape)) - return decoder_input_data, decoder_target_data - - -def load_encoder_inputs(encoder_np_vecs='train_body_vecs.npy'): - """ - Load variables & data that are inputs to encoder. - - Parameters - ---------- - encoder_np_vecs : str - filename of serialized numpy.array of encoder input (issue title) - - Returns - ------- - encoder_input_data : numpy.array - The issue body - doc_length : int - The standard document length of the input for the encoder after padding - the shape of this array will be (num_examples, doc_length) - - """ - vectorized_body = np.load(encoder_np_vecs) - # Encoder input is simply the body of the issue text - encoder_input_data = vectorized_body - doc_length = encoder_input_data.shape[1] - print('Shape of encoder input: {}'.format(encoder_input_data.shape)) - return encoder_input_data, doc_length - - -def viz_model_architecture(model): - """Visualize model architecture in Jupyter notebook.""" - display(SVG(model_to_dot(model).create(prog='dot', format='svg'))) - - -def free_gpu_mem(): - """Attempt to free gpu memory.""" - K.get_session().close() - cfg = K.tf.ConfigProto() - cfg.gpu_options.allow_growth = True - K.set_session(K.tf.Session(config=cfg)) - - -def test_gpu(): - """Run a toy computation task in tensorflow to test GPU.""" - config = tf.ConfigProto() - config.gpu_options.allow_growth = True - session = tf.Session(config=config) - hello = tf.constant('Hello, TensorFlow!') - print(session.run(hello)) - - -def plot_model_training_history(history_object): - """Plots model train vs. validation loss.""" - plt.title('model accuracy') - plt.ylabel('accuracy') - plt.xlabel('epoch') - plt.plot(history_object.history['loss']) - plt.plot(history_object.history['val_loss']) - plt.legend(['train', 'test'], loc='upper left') - plt.show() - - -def extract_encoder_model(model): - """ - Extract the encoder from the original Sequence to Sequence Model. - - Returns a keras model object that has one input (body of issue) and one - output (encoding of issue, which is the last hidden state). - - Input: - ----- - model: keras model object - - Returns: - ----- - keras model object - - """ - encoder_model = model.get_layer('Encoder-Model') - return encoder_model - - -def extract_decoder_model(model): - """ - Extract the decoder from the original model. - - Inputs: - ------ - model: keras model object - - Returns: - ------- - A Keras model object with the following inputs and outputs: - - Inputs of Keras Model That Is Returned: - 1: the embedding index for the last predicted word or the indicator - 2: the last hidden state, or in the case of the first word the hidden state from the encoder - - Outputs of Keras Model That Is Returned: - 1. Prediction (class probabilities) for the next word - 2. The hidden state of the decoder, to be fed back into the decoder at the next time step - - Implementation Notes: - ---------------------- - Must extract relevant layers and reconstruct part of the computation graph - to allow for different inputs as we are not going to use teacher forcing at - inference time. - - """ - # the latent dimension is the same throughout the architecture so we are going to - # cheat and grab the latent dimension of the embedding because that is the same as what is - # output from the decoder - latent_dim = model.get_layer('Decoder-Word-Embedding').output_shape[-1] - - # Reconstruct the input into the decoder - decoder_inputs = model.get_layer('Decoder-Input').input - dec_emb = model.get_layer('Decoder-Word-Embedding')(decoder_inputs) - dec_bn = model.get_layer('Decoder-Batchnorm-1')(dec_emb) - - # Instead of setting the intial state from the encoder and forgetting about it, during inference - # we are not doing teacher forcing, so we will have to have a feedback loop from predictions back - # into the GRU, thus we define this input layer for the state so we can add this capability - gru_inference_state_input = Input(shape=(latent_dim,), name='hidden_state_input') - - # we need to reuse the weights that is why we are getting this - # If you inspect the decoder GRU that we created for training, it will take as input - # 2 tensors -> (1) is the embedding layer output for the teacher forcing - # (which will now be the last step's prediction, and will be _start_ on the - # first time step) - # (2) is the state, which we will initialize with the encoder on the first time step - # but then grab the state after the first prediction and feed that back in again. - gru_out, gru_state_out = model.get_layer('Decoder-GRU')([dec_bn, gru_inference_state_input]) - - # Reconstruct dense layers - dec_bn2 = model.get_layer('Decoder-Batchnorm-2')(gru_out) - dense_out = model.get_layer('Final-Output-Dense')(dec_bn2) - decoder_model = Model([decoder_inputs, gru_inference_state_input], - [dense_out, gru_state_out]) - return decoder_model - - -class Seq2Seq_Inference(object): - - # pylint: disable=too-many-instance-attributes - - def __init__(self, - encoder_preprocessor, - decoder_preprocessor, - seq2seq_model): - - self.pp_body = encoder_preprocessor - self.pp_title = decoder_preprocessor - self.seq2seq_model = seq2seq_model - self.encoder_model = extract_encoder_model(seq2seq_model) - self.decoder_model = extract_decoder_model(seq2seq_model) - self.default_max_len_title = self.pp_title.padding_maxlen - self.nn = None - self.rec_df = None - - def generate_issue_title(self, - raw_input_text, - max_len_title=None): - """ - Use the seq2seq model to generate a title given the body of an issue. - - Inputs - ------ - raw_input: str - The body of the issue text as an input string - - max_len_title: int (optional) - The maximum length of the title the model will generate - - """ - if max_len_title is None: - max_len_title = self.default_max_len_title - # get the encoder's features for the decoder - raw_tokenized = self.pp_body.transform([raw_input_text]) - body_encoding = self.encoder_model.predict(raw_tokenized) - # we want to save the encoder's embedding before its updated by decoder - # because we can use that as an embedding for other tasks. - original_body_encoding = body_encoding - state_value = np.array(self.pp_title.token2id['_start_']).reshape(1, 1) - - decoded_sentence = [] - stop_condition = False - while not stop_condition: - preds, st = self.decoder_model.predict([state_value, body_encoding]) - - # We are going to ignore indices 0 (padding) and indices 1 (unknown) - # Argmax will return the integer index corresponding to the - # prediction + 2 b/c we chopped off first two - pred_idx = np.argmax(preds[:, :, 2:]) + 2 - - # retrieve word from index prediction - pred_word_str = self.pp_title.id2token[pred_idx] - - if pred_word_str == '_end_' or len(decoded_sentence) >= max_len_title: - stop_condition = True - break - decoded_sentence.append(pred_word_str) - - # update the decoder for the next word - body_encoding = st - state_value = np.array(pred_idx).reshape(1, 1) - - return original_body_encoding, ' '.join(decoded_sentence) - - - def print_example(self, - i, - body_text, - title_text, - url, - threshold): - """ - Prints an example of the model's prediction for manual inspection. - """ - if i: - print('\n\n==============================================') - print('============== Example # {} =================\n'.format(i)) - - if url: - print(url) - - print("Issue Body:\n {} \n".format(body_text)) - - if title_text: - print("Original Title:\n {}".format(title_text)) - - emb, gen_title = self.generate_issue_title(body_text) - print("\n****** Machine Generated Title (Prediction) ******:\n {}".format(gen_title)) - - if self.nn: - # return neighbors and distances - n, d = self.nn.get_nns_by_vector(emb.flatten(), n=4, - include_distances=True) - neighbors = n[1:] - dist = d[1:] - - if min(dist) <= threshold: - cols = ['issue_url', 'issue_title', 'body'] - dfcopy = self.rec_df.iloc[neighbors][cols].copy(deep=True) - dfcopy['dist'] = dist - similar_issues_df = dfcopy.query('dist <= {}'.format(threshold)) - - print("\n**** Similar Issues (using encoder embedding) ****:\n") - display(similar_issues_df) - - - def demo_model_predictions(self, - n, - issue_df, - threshold=1): - """ - Pick n random Issues and display predictions. - - Input: - ------ - n : int - Number of issues to display from issue_df - issue_df : pandas DataFrame - DataFrame that contains two columns: `body` and `issue_title`. - threshold : float - distance threshold for recommendation of similar issues. - - Returns: - -------- - None - Prints the original issue body and the model's prediction. - """ - # Extract body and title from DF - body_text = issue_df.body.tolist() - title_text = issue_df.issue_title.tolist() - url = issue_df.issue_url.tolist() - - demo_list = np.random.randint(low=1, high=len(body_text), size=n) - for i in demo_list: - self.print_example(i, - body_text=body_text[i], - title_text=title_text[i], - url=url[i], - threshold=threshold) - - def prepare_recommender(self, vectorized_array, original_df): - """ - Use the annoy library to build recommender - - Parameters - ---------- - vectorized_array : List[List[int]] - This is the list of list of integers that represents your corpus - that is fed into the seq2seq model for training. - original_df : pandas.DataFrame - This is the original dataframe that has the columns - ['issue_url', 'issue_title', 'body'] - - Returns - ------- - annoy.AnnoyIndex object (see https://github.com/spotify/annoy) - """ - self.rec_df = original_df - emb = self.encoder_model.predict(x=vectorized_array, - batch_size=vectorized_array.shape[0]//200) - - f = emb.shape[1] - self.nn = AnnoyIndex(f) - logging.warning('Adding embeddings') - for i in tqdm(range(len(emb))): - self.nn.add_item(i, emb[i]) - logging.warning('Building trees for similarity lookup.') - self.nn.build(50) - return self.nn - - def set_recsys_data(self, original_df): - self.rec_df = original_df - - def set_recsys_annoyobj(self, annoyobj): - self.nn = annoyobj - - def evaluate_model(self, holdout_bodies, holdout_titles): - """ - Method for calculating BLEU Score. - - Parameters - ---------- - holdout_bodies : List[str] - These are the issue bodies that we want to summarize - holdout_titles : List[str] - This is the ground truth we are trying to predict --> issue titles - - Returns - ------- - bleu : float - The BLEU Score - - """ - actual, predicted = list(), list() - assert len(holdout_bodies) == len(holdout_titles) - num_examples = len(holdout_bodies) - - logging.warning('Generating predictions.') - # step over the whole set TODO: parallelize this - for i in tqdm_notebook(range(num_examples)): - _, yhat = self.generate_issue_title(holdout_bodies[i]) - - actual.append(self.pp_title.process_text([holdout_titles[i]])[0]) - predicted.append(self.pp_title.process_text([yhat])[0]) - - # calculate BLEU score - logging.warning('Calculating BLEU.') - #must be careful with nltk api for corpus_bleu!, - # expects List[List[List[str]]] for ground truth, using List[List[str]] will give you - # erroneous results. - bleu = corpus_bleu([[a] for a in actual], predicted) - return bleu diff --git a/examples/GKEDemo/docker-image/train.py b/examples/GKEDemo/docker-image/train.py deleted file mode 100644 index ec290915055..00000000000 --- a/examples/GKEDemo/docker-image/train.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Train the github-issue-summarization model -train.py trains the github-issue-summarization model. - -It reads the input data from GCS in a zip file format. ---input_data_gcs_bucket and --input_data_gcs_path specify -the location of input data. - -It write the model back to GCS. ---output_model_gcs_bucket and --output_model_gcs_path specify -the location of output. - -It also has parameters which control the training like ---learning_rate and --sample_size - -""" -import argparse -import logging -import zipfile -import urllib - -from google.cloud import storage # pylint: disable=no-name-in-module -import dill as dpickle -import numpy as np -import pandas as pd -from keras import optimizers -from keras.layers import GRU, BatchNormalization, Dense, Embedding, Input -from keras.models import Model -from sklearn.model_selection import train_test_split - -from ktext.preprocess import processor -from seq2seq_utils import load_encoder_inputs, load_text_processor - -def main(): # pylint: disable=too-many-statements - # Parsing flags. - parser = argparse.ArgumentParser() - parser.add_argument("--sample_size", type=int, default=2000000) - parser.add_argument("--learning_rate", default="0.001") - - parser.add_argument( - "--input_data_gcs_bucket", type=str, default="") - parser.add_argument( - "--input_data_gcs_path", - type=str, - default="") - - parser.add_argument( - "--output_model_gcs_bucket", type=str, default="") - parser.add_argument( - "--output_model_gcs_path", - type=str, - default="") - - parser.add_argument( - "--output_body_preprocessor_dpkl", - type=str, - default="body_preprocessor.dpkl") - parser.add_argument( - "--output_title_preprocessor_dpkl", - type=str, - default="title_preprocessor.dpkl") - parser.add_argument( - "--output_train_title_vecs_npy", type=str, default="train_title_vecs.npy") - parser.add_argument( - "--output_train_body_vecs_npy", type=str, default="train_body_vecs.npy") - parser.add_argument("--output_model_h5", type=str, default="output_model.h5") - - args = parser.parse_args() - logging.info(args) - - learning_rate = float(args.learning_rate) - - pd.set_option('display.max_colwidth', 500) - print("Download iput file") - if args.input_data_gcs_bucket != "" and args.input_data_gcs_path != "": - bucket = storage.Bucket(storage.Client(), args.input_data_gcs_bucket) - storage.Blob(args.input_data_gcs_path, - bucket).download_to_filename('github-issues.zip') - else: - urllib.request.urlretrieve("https://storage.googleapis.com/kubeflow-examples/github-issue-summarization-data/github-issues.zip",'github-issues.zip') - - print("unzip iput file") - zip_ref = zipfile.ZipFile('github-issues.zip', 'r') - zip_ref.extractall('.') - zip_ref.close() - - # Read in data sample 2M rows (for speed of tutorial) - traindf, testdf = train_test_split( - pd.read_csv('github_issues.csv').sample(n=args.sample_size), test_size=.10) - - # Print stats about the shape of the data. - logging.info('Train: %d rows %d columns', traindf.shape[0], traindf.shape[1]) - logging.info('Test: %d rows %d columns', testdf.shape[0], testdf.shape[1]) - - train_body_raw = traindf.body.tolist() - train_title_raw = traindf.issue_title.tolist() - - # Clean, tokenize, and apply padding / truncating such that each document - # length = 70. Also, retain only the top 8,000 words in the vocabulary and set - # the remaining words to 1 which will become common index for rare words. - body_pp = processor(keep_n=8000, padding_maxlen=70) - train_body_vecs = body_pp.fit_transform(train_body_raw) - - logging.info('Example original body: %s', train_body_raw[0]) - logging.info('Example body after pre-processing: %s', train_body_vecs[0]) - - # Instantiate a text processor for the titles, with some different parameters. - title_pp = processor( - append_indicators=True, keep_n=4500, padding_maxlen=12, padding='post') - - # process the title data - train_title_vecs = title_pp.fit_transform(train_title_raw) - - logging.info('Example original title: %s', train_title_raw[0]) - logging.info('Example title after pre-processing: %s', train_title_vecs[0]) - - # Save the preprocessor. - with open(args.output_body_preprocessor_dpkl, 'wb') as f: - dpickle.dump(body_pp, f) - - with open(args.output_title_preprocessor_dpkl, 'wb') as f: - dpickle.dump(title_pp, f) - - # Save the processed data. - np.save(args.output_train_title_vecs_npy, train_title_vecs) - np.save(args.output_train_body_vecs_npy, train_body_vecs) - - _, doc_length = load_encoder_inputs( - args.output_train_body_vecs_npy) - - num_encoder_tokens, body_pp = load_text_processor( - args.output_body_preprocessor_dpkl) - num_decoder_tokens, title_pp = load_text_processor( - args.output_title_preprocessor_dpkl) - - # Arbitrarly set latent dimension for embedding and hidden units - latent_dim = 300 - - ############### - # Encoder Model. - ############### - encoder_inputs = Input(shape=(doc_length,), name='Encoder-Input') - - # Word embeding for encoder (ex: Issue Body) - x = Embedding( - num_encoder_tokens, latent_dim, name='Body-Word-Embedding', - mask_zero=False)(encoder_inputs) - x = BatchNormalization(name='Encoder-Batchnorm-1')(x) - - # We do not need the `encoder_output` just the hidden state. - _, state_h = GRU(latent_dim, return_state=True, name='Encoder-Last-GRU')(x) - - # Encapsulate the encoder as a separate entity so we can just - # encode without decoding if we want to. - encoder_model = Model( - inputs=encoder_inputs, outputs=state_h, name='Encoder-Model') - - seq2seq_encoder_out = encoder_model(encoder_inputs) - - ################ - # Decoder Model. - ################ - decoder_inputs = Input( - shape=(None,), name='Decoder-Input') # for teacher forcing - - # Word Embedding For Decoder (ex: Issue Titles) - dec_emb = Embedding( - num_decoder_tokens, - latent_dim, - name='Decoder-Word-Embedding', - mask_zero=False)(decoder_inputs) - dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb) - - # Set up the decoder, using `decoder_state_input` as initial state. - decoder_gru = GRU( - latent_dim, return_state=True, return_sequences=True, name='Decoder-GRU') - decoder_gru_output, _ = decoder_gru(dec_bn, initial_state=seq2seq_encoder_out) - x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output) - - # Dense layer for prediction - decoder_dense = Dense( - num_decoder_tokens, activation='softmax', name='Final-Output-Dense') - decoder_outputs = decoder_dense(x) - - ################ - # Seq2Seq Model. - ################ - - seq2seq_Model = Model([encoder_inputs, decoder_inputs], decoder_outputs) - - seq2seq_Model.compile( - optimizer=optimizers.Nadam(lr=learning_rate), - loss='sparse_categorical_crossentropy') - - seq2seq_Model.summary() - - ############# - # Save model. - ############# - seq2seq_Model.save(args.output_model_h5) - - ###################### - # Upload model to GCS. - ###################### - if args.output_model_gcs_bucket != "": - bucket = storage.Bucket(storage.Client(), args.output_model_gcs_bucket) - storage.Blob(args.output_model_gcs_path, bucket).upload_from_filename( - args.output_model_h5) - - -if __name__ == '__main__': - main() diff --git a/examples/GKEDemo/down.sh b/examples/GKEDemo/down.sh deleted file mode 100755 index 0ffdf8c3900..00000000000 --- a/examples/GKEDemo/down.sh +++ /dev/null @@ -1,3 +0,0 @@ -#/bin/bash -set -x -kubectl delete ns katib diff --git a/examples/GKEDemo/git-issue-summarize-demo.go b/examples/GKEDemo/git-issue-summarize-demo.go deleted file mode 100644 index aea10dbaac6..00000000000 --- a/examples/GKEDemo/git-issue-summarize-demo.go +++ /dev/null @@ -1,208 +0,0 @@ -package main - -import ( - "context" - "flag" - "log" - "time" - - "github.com/kubeflow/katib/pkg/api" - "google.golang.org/grpc" -) - -var studyConfig = api.StudyConfig{ - Name: "grid-demo", - Owner: "katib", - OptimizationType: api.OptimizationType_MAXIMIZE, - OptimizationGoal: 0.99, - ObjectiveValueName: "Validation-accuracy", - Metrics: []string{ - "accuracy", - }, - ParameterConfigs: &api.StudyConfig_ParameterConfigs{ - Configs: []*api.ParameterConfig{ - &api.ParameterConfig{ - Name: "--learning_rate", - ParameterType: api.ParameterType_DOUBLE, - Feasible: &api.FeasibleSpace{ - Min: "0.005", - Max: "0.5", - }, - }, - }, - }, -} - -var workerConfig = api.WorkerConfig{ - Image: "yujioshima/tf-job-issue-summarization:latest", - Command: []string{ - "python", - "/workdir/train.py", - "--sample_size", - "20000", - // "--input_data_gcs_bucket", - // "katib-gi-example", - // "--input_data_gcs_path", - // "github-issue-summarization-data/github-issues.zip", - // "--output_model_gcs_bucket", - // "katib-gi-example", - }, - Gpu: 0, - Scheduler: "default-scheduler", -} - -var gridConfig = []*api.SuggestionParameter{ - &api.SuggestionParameter{ - Name: "DefaultGrid", - Value: "4", - }, - &api.SuggestionParameter{ - Name: "--learning_rate", - Value: "2", - }, -} - -var managerAddr = flag.String("s", "127.0.0.1:6789", "Endpoint of manager default 127.0.0.1:6789") - -func main() { - flag.Parse() - conn, err := grpc.Dial(*managerAddr, grpc.WithInsecure()) - if err != nil { - log.Fatalf("could not connect: %v", err) - } - defer conn.Close() - ctx := context.Background() - c := api.NewManagerClient(conn) - createStudyreq := &api.CreateStudyRequest{ - StudyConfig: &studyConfig, - } - createStudyreply, err := c.CreateStudy(ctx, createStudyreq) - if err != nil { - log.Fatalf("StudyConfig Error %v", err) - } - studyId := createStudyreply.StudyId - log.Printf("Study ID %s", studyId) - getStudyreq := &api.GetStudyRequest{ - StudyId: studyId, - } - getStudyReply, err := c.GetStudy(ctx, getStudyreq) - if err != nil { - log.Fatalf("GetConfig Error %v", err) - } - log.Printf("Study ID %s StudyConf%v", studyId, getStudyReply.StudyConfig) - setSuggesitonParameterRequest := &api.SetSuggestionParametersRequest{ - StudyId: studyId, - SuggestionAlgorithm: "grid", - SuggestionParameters: gridConfig, - } - setSuggesitonParameterReply, err := c.SetSuggestionParameters(ctx, setSuggesitonParameterRequest) - if err != nil { - log.Fatalf("SetConfig Error %v", err) - } - log.Printf("Grid Prameter ID %s", setSuggesitonParameterReply.ParamId) - getGridSuggestRequest := &api.GetSuggestionsRequest{ - StudyId: studyId, - SuggestionAlgorithm: "grid", - RequestNumber: 0, - //RequestNumber=0 means get all grids. - ParamId: setSuggesitonParameterReply.ParamId, - } - getGridSuggestReply, err := c.GetSuggestions(ctx, getGridSuggestRequest) - if err != nil { - log.Fatalf("GetSuggestion Error %v", err) - } - log.Println("Get Grid Suggestions:") - for _, t := range getGridSuggestReply.Trials { - log.Printf("%v", t) - } - workerIds := make([]string, len(getGridSuggestReply.Trials)) - workerParameter := make(map[string][]*api.Parameter) - for i, t := range getGridSuggestReply.Trials { - ws := workerConfig - rtr := &api.RunTrialRequest{ - StudyId: studyId, - TrialId: t.TrialId, - Runtime: "kubernetes", - WorkerConfig: &ws, - } - rtr.WorkerConfig.Command = append(rtr.WorkerConfig.Command, "--output_model_gcs_path") - rtr.WorkerConfig.Command = append(rtr.WorkerConfig.Command, "github-issue-summarization-data/"+t.TrialId+"output_model.h5") - for _, p := range t.ParameterSet { - rtr.WorkerConfig.Command = append(rtr.WorkerConfig.Command, p.Name) - rtr.WorkerConfig.Command = append(rtr.WorkerConfig.Command, p.Value) - } - workerReply, err := c.RunTrial(ctx, rtr) - if err != nil { - log.Fatalf("RunTrial Error %v", err) - } - workerIds[i] = workerReply.WorkerId - workerParameter[workerReply.WorkerId] = t.ParameterSet - saveModelRequest := &api.SaveModelRequest{ - Model: &api.ModelInfo{ - StudyName: studyConfig.Name, - WorkerId: workerReply.WorkerId, - Parameters: t.ParameterSet, - Metrics: []*api.Metrics{}, - ModelPath: "pvc:/Path/to/Model", - }, - DataSet: &api.DataSetInfo{ - Name: "Mnist", - Path: "/path/to/data", - }, - } - _, err = c.SaveModel(ctx, saveModelRequest) - if err != nil { - log.Fatalf("SaveModel Error %v", err) - } - log.Printf("WorkerID %s start\n", workerReply.WorkerId) - } - for true { - time.Sleep(10 * time.Second) - getMetricsRequest := &api.GetMetricsRequest{ - StudyId: studyId, - WorkerIds: workerIds, - } - getMetricsReply, err := c.GetMetrics(ctx, getMetricsRequest) - if err != nil { - log.Printf("GetMetErr %v", err) - continue - } - for _, mls := range getMetricsReply.MetricsLogSets { - if len(mls.MetricsLogs) > 0 { - //Only Metrics can be updated. - saveModelRequest := &api.SaveModelRequest{ - Model: &api.ModelInfo{ - StudyName: studyConfig.Name, - WorkerId: mls.WorkerId, - Metrics: []*api.Metrics{}, - }, - } - for _, ml := range mls.MetricsLogs { - if len(ml.Values) > 0 { - log.Printf("WorkerID %s :\t Metrics Name %s Value %v", mls.WorkerId, ml.Name, ml.Values[len(ml.Values)-1]) - saveModelRequest.Model.Metrics = append(saveModelRequest.Model.Metrics, &api.Metrics{Name: ml.Name, Value: ml.Values[len(ml.Values)-1]}) - } - } - _, err = c.SaveModel(ctx, saveModelRequest) - if err != nil { - log.Fatalf("SaveModel Error %v", err) - } - } - } - getWorkerRequest := &api.GetWorkersRequest{StudyId: studyId} - getWorkerReply, err := c.GetWorkers(ctx, getWorkerRequest) - if err != nil { - log.Fatalf("GetWorker Error %v", err) - } - completeCount := 0 - for _, w := range getWorkerReply.Workers { - if w.Status == api.State_COMPLETED { - completeCount++ - } - } - if completeCount == len(getWorkerReply.Workers) { - log.Printf("All Worker Completed!") - break - } - } -} diff --git a/examples/GKEDemo/manifests/0-namespace.yaml b/examples/GKEDemo/manifests/0-namespace.yaml deleted file mode 100644 index 9bac532fac9..00000000000 --- a/examples/GKEDemo/manifests/0-namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: katib diff --git a/examples/GKEDemo/manifests/modeldb/backend/deployment.yaml b/examples/GKEDemo/manifests/modeldb/backend/deployment.yaml deleted file mode 100644 index bc7fd1dc9f6..00000000000 --- a/examples/GKEDemo/manifests/modeldb/backend/deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: modeldb-backend - namespace: katib - labels: - app: modeldb - component: backend -spec: - replicas: 1 - template: - metadata: - name: modeldb-backend - labels: - app: modeldb - component: backend - spec: - containers: - - name: modeldb-backend - image: mitdbg/modeldb-backend:latest - args: - - 'modeldb-db' - ports: - - name: api - containerPort: 6543 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/modeldb/backend/service.yaml b/examples/GKEDemo/manifests/modeldb/backend/service.yaml deleted file mode 100644 index 71d9d5f7fe8..00000000000 --- a/examples/GKEDemo/manifests/modeldb/backend/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: modeldb-backend - namespace: katib - labels: - app: modeldb - component: backend -spec: - type: ClusterIP - ports: - - port: 6543 - protocol: TCP - name: api - selector: - app: modeldb - component: backend diff --git a/examples/GKEDemo/manifests/modeldb/db/deployment.yaml b/examples/GKEDemo/manifests/modeldb/db/deployment.yaml deleted file mode 100644 index eaa83fa2db6..00000000000 --- a/examples/GKEDemo/manifests/modeldb/db/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: modeldb-db - namespace: katib - labels: - app: modeldb - component: db -spec: - replicas: 1 - template: - metadata: - name: modeldb-db - labels: - app: modeldb - component: db - spec: - containers: - - name: modeldb-db - image: mongo:3.4 - ports: - - name: dbapi - containerPort: 27017 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/modeldb/db/service.yaml b/examples/GKEDemo/manifests/modeldb/db/service.yaml deleted file mode 100644 index 966fb4155ea..00000000000 --- a/examples/GKEDemo/manifests/modeldb/db/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: modeldb-db - namespace: katib - labels: - app: modeldb - component: db -spec: - type: ClusterIP - ports: - - port: 27017 - protocol: TCP - name: dbapi - selector: - app: modeldb - component: db diff --git a/examples/GKEDemo/manifests/modeldb/frontend/deployment.yaml b/examples/GKEDemo/manifests/modeldb/frontend/deployment.yaml deleted file mode 100644 index 3c1e679c0e4..00000000000 --- a/examples/GKEDemo/manifests/modeldb/frontend/deployment.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: modeldb-frontend - namespace: katib - labels: - app: modeldb - component: frontend -spec: - replicas: 1 - template: - metadata: - name: modeldb-frontend - labels: - app: modeldb - component: frontend - spec: - containers: - - name: modeldb-frontend - image: katib/katib-frontend - imagePullPolicy: Always - args: - - 'modeldb-backend' - env: - - name: ROOT_PATH - value: "" - ports: - - name: webapi - containerPort: 3000 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/modeldb/frontend/ingress.yaml b/examples/GKEDemo/manifests/modeldb/frontend/ingress.yaml deleted file mode 100644 index fbb4eb24819..00000000000 --- a/examples/GKEDemo/manifests/modeldb/frontend/ingress.yaml +++ /dev/null @@ -1,9 +0,0 @@ - apiVersion: extensions/v1beta1 - kind: Ingress - metadata: - name: katib-ui - namespace: katib - spec: - backend: - serviceName: modeldb-frontend - servicePort: 80 diff --git a/examples/GKEDemo/manifests/modeldb/frontend/service.yaml b/examples/GKEDemo/manifests/modeldb/frontend/service.yaml deleted file mode 100644 index c94d0a4de37..00000000000 --- a/examples/GKEDemo/manifests/modeldb/frontend/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: modeldb-frontend - namespace: katib - labels: - app: modeldb - component: frontend -spec: - type: ClusterIP - ports: - - port: 3000 - protocol: TCP - name: ui - selector: - app: modeldb - component: frontend diff --git a/examples/GKEDemo/manifests/vizier/core/deployment.yaml b/examples/GKEDemo/manifests/vizier/core/deployment.yaml deleted file mode 100644 index cc0578ef069..00000000000 --- a/examples/GKEDemo/manifests/vizier/core/deployment.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: vizier-core - namespace: katib - labels: - app: vizier - component: core -spec: - replicas: 1 - template: - metadata: - name: vizier-core - labels: - app: vizier - component: core - spec: - serviceAccountName: vizier-core - containers: - - name: vizier-core - image: katib/vizier-core - command: - - './vizier-manager' - - "-w" - - "kubernetes" - - "-i" - - "k-cluster.osrg.net" - ports: - - name: api - containerPort: 6789 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/vizier/core/ingress.yaml b/examples/GKEDemo/manifests/vizier/core/ingress.yaml deleted file mode 100644 index e0c307146b5..00000000000 --- a/examples/GKEDemo/manifests/vizier/core/ingress.yaml +++ /dev/null @@ -1,9 +0,0 @@ - apiVersion: extensions/v1beta1 - kind: Ingress - metadata: - name: vizier-core - namespace: katib - spec: - backend: - serviceName: vizier-core - servicePort: 6789 diff --git a/examples/GKEDemo/manifests/vizier/core/rbac.yaml b/examples/GKEDemo/manifests/vizier/core/rbac.yaml deleted file mode 100644 index a0858e752af..00000000000 --- a/examples/GKEDemo/manifests/vizier/core/rbac.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: vizier-core -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: vizier-core -subjects: -- kind: ServiceAccount - name: vizier-core - namespace: katib ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: vizier-core -rules: - - apiGroups: [""] - resources: ["pods", "nodes", "nodes/*", "pods/log", "pods/status", "services", "persistentvolumes", "persistentvolumes/status","persistentvolumeclaims","persistentvolumeclaims/status"] - verbs: ["*"] - - apiGroups: ["batch"] - resources: ["jobs", "jobs/status"] - verbs: ["*"] - - verbs: ["*"] - apiGroups: ["extensions"] - resources: ["ingresses","ingresses/status","deployments","deployments/status"] - - verbs: ["*"] - apiGroups: [""] - resources: ["services"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: vizier-core - namespace: katib diff --git a/examples/GKEDemo/manifests/vizier/core/service.yaml b/examples/GKEDemo/manifests/vizier/core/service.yaml deleted file mode 100644 index ca171d819a4..00000000000 --- a/examples/GKEDemo/manifests/vizier/core/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: vizier-core - namespace: katib - labels: - app: vizier - component: core -spec: - type: ClusterIP - ports: - - port: 6789 - protocol: TCP - name: api - selector: - app: vizier - component: core diff --git a/examples/GKEDemo/manifests/vizier/db/deployment.yaml b/examples/GKEDemo/manifests/vizier/db/deployment.yaml deleted file mode 100644 index bd4a9ef1958..00000000000 --- a/examples/GKEDemo/manifests/vizier/db/deployment.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: vizier-db - namespace: katib - labels: - app: vizier - component: db -spec: - replicas: 1 - template: - metadata: - name: vizier-db - labels: - app: vizier - component: db - spec: - containers: - - name: vizier-db - image: mysql:8.0.3 - env: - - name: MYSQL_ROOT_PASSWORD - value: "test" - - name: MYSQL_ALLOW_EMPTY_PASSWORD - value: "true" - - name: MYSQL_DATABASE - value: "vizier" - ports: - - name: dbapi - containerPort: 3306 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/vizier/db/service.yaml b/examples/GKEDemo/manifests/vizier/db/service.yaml deleted file mode 100644 index dde0b2c31a0..00000000000 --- a/examples/GKEDemo/manifests/vizier/db/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: vizier-db - namespace: katib - labels: - app: vizier - component: db -spec: - type: ClusterIP - ports: - - port: 3306 - protocol: TCP - name: dbapi - selector: - app: vizier - component: db diff --git a/examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/deployment.yaml b/examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/deployment.yaml deleted file mode 100644 index 7a4ef2c5425..00000000000 --- a/examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: vizier-earlystopping-medianstopping - namespace: katib - labels: - app: vizier - component: earlystopping-medianstopping -spec: - replicas: 1 - template: - metadata: - name: vizier-earlystopping-medianstopping - labels: - app: vizier - component: earlystopping-medianstopping - spec: - containers: - - name: vizier-earlystopping-medianstopping - image: katib/earlystopping-medianstopping - ports: - - name: api - containerPort: 6789 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/service.yaml b/examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/service.yaml deleted file mode 100644 index dd6aa39c465..00000000000 --- a/examples/GKEDemo/manifests/vizier/earlystopping/medianstopping/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: vizier-earlystopping-medianstopping - namespace: katib - labels: - app: vizier - component: earlystopping-medianstopping -spec: - type: ClusterIP - ports: - - port: 6789 - protocol: TCP - name: api - selector: - app: vizier - component: earlystopping-medianstopping diff --git a/examples/GKEDemo/manifests/vizier/suggestion/grid/deployment.yaml b/examples/GKEDemo/manifests/vizier/suggestion/grid/deployment.yaml deleted file mode 100644 index 707f170ddf9..00000000000 --- a/examples/GKEDemo/manifests/vizier/suggestion/grid/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: vizier-suggestion-grid - namespace: katib - labels: - app: vizier - component: suggestion-grid -spec: - replicas: 1 - template: - metadata: - name: vizier-suggestion-grid - labels: - app: vizier - component: suggestion-grid - spec: - containers: - - name: vizier-suggestion-grid - image: katib/suggestion-grid - ports: - - name: api - containerPort: 6789 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/vizier/suggestion/grid/service.yaml b/examples/GKEDemo/manifests/vizier/suggestion/grid/service.yaml deleted file mode 100644 index 4a1cf6b64a0..00000000000 --- a/examples/GKEDemo/manifests/vizier/suggestion/grid/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: vizier-suggestion-grid - namespace: katib - labels: - app: vizier - component: suggestion-grid -spec: - type: ClusterIP - ports: - - port: 6789 - protocol: TCP - name: api - selector: - app: vizier - component: suggestion-grid diff --git a/examples/GKEDemo/manifests/vizier/suggestion/random/deployment.yaml b/examples/GKEDemo/manifests/vizier/suggestion/random/deployment.yaml deleted file mode 100644 index 187cac3998b..00000000000 --- a/examples/GKEDemo/manifests/vizier/suggestion/random/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: vizier-suggestion-random - namespace: katib - labels: - app: vizier - component: suggestion-random -spec: - replicas: 1 - template: - metadata: - name: vizier-suggestion-random - labels: - app: vizier - component: suggestion-random - spec: - containers: - - name: vizier-suggestion-random - image: katib/suggestion-random - ports: - - name: api - containerPort: 6789 -# resources: -# requests: -# cpu: 500m -# memory: 500M -# limits: -# cpu: 500m -# memory: 500M diff --git a/examples/GKEDemo/manifests/vizier/suggestion/random/service.yaml b/examples/GKEDemo/manifests/vizier/suggestion/random/service.yaml deleted file mode 100644 index 463cdca1da0..00000000000 --- a/examples/GKEDemo/manifests/vizier/suggestion/random/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: vizier-suggestion-random - namespace: katib - labels: - app: vizier - component: suggestion-random -spec: - type: ClusterIP - ports: - - port: 6789 - protocol: TCP - name: api - selector: - app: vizier - component: suggestion-random