Modify SDK

kubeflow · Nov 27, 2020 · 68ba3e6 · 68ba3e6
1 parent 0eaf409
commit 68ba3e6
Show file tree

Hide file tree

Showing 31 changed files with 278 additions and 95 deletions.
diff --git a/hack/gen-python-sdk/gen-sdk.sh b/hack/gen-python-sdk/gen-sdk.sh
@@ -36,6 +36,7 @@ for VERSION in ${KATIB_VERSIONS[@]}; do
     TMP_PATH=${TMP_CODEGEN_PATH/KATIB_VERSION/$VERSION}
     java -jar ${SWAGGER_CODEGEN_JAR} generate -i ${SWAGGER_FILE} -l python -o ${TMP_PATH} -c ${SWAGGER_CODEGEN_CONF}
 
+    # Run post gen script.
     python ${POST_GEN_PYTHON_HANDLER} ${TMP_PATH} ${SDK_OUTPUT_PATH}/${VERSION}
 done
 

diff --git a/hack/gen-python-sdk/post_gen.py b/hack/gen-python-sdk/post_gen.py
@@ -2,6 +2,11 @@
 import shutil
 import sys
 
+IGNORE_LINES = [
+    "from kubeflow.katib.models.v1_unstructured_unstructured import V1UnstructuredUnstructured",
+    "from kubeflow.katib.models.v1_time import V1Time"
+]
+
 
 def _rewrite_helper(input_file, output_file, rewrite_rules):
     rules = rewrite_rules or []
@@ -11,9 +16,12 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
             line = f.readline()
             if not line:
                 break
+            # Apply rewrite rules to the line.
             for rule in rules:
                 line = rule(line)
-            lines.append(line)
+            # Remove ignored lines.
+            if not any(l in line for l in IGNORE_LINES):
+                lines.append(line)
 
     # Add Katib client to init file
     if (output_file == "sdk/python/v1beta1/kubeflow/katib/__init__.py"):
@@ -30,8 +38,29 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
 def update_python_sdk(src, dest, versions=('v1beta1')):
     # tiny transformers to refine generated codes
     rewrite_rules = [
+        # Models rules.
         lambda l: l.replace('import katib', 'import kubeflow.katib'),
         lambda l: l.replace('from katib', 'from kubeflow.katib'),
+        # Doc rules.
+        lambda l: l.replace('[**datetime**](V1Time.md)', '**datetime**'),
+        lambda l: l.replace('[**dict()**](V1UnstructuredUnstructured.md)', '**dict()**'),
+
+        lambda l: l.replace('[**V1Container**](V1Container.md)',
+                            '[**V1Container**](https://github.com/kubernetes-client/'
+                            'python/blob/master/kubernetes/docs/V1Container.md)'),
+
+        lambda l: l.replace('[**V1ObjectMeta**](V1ObjectMeta.md)',
+                            '[**V1ObjectMeta**](https://github.com/kubernetes-client/'
+                            'python/blob/master/kubernetes/docs/V1ObjectMeta.md)'),
+
+        lambda l: l.replace('[**V1ListMeta**](V1ListMeta.md)',
+                            '[**V1ListMeta**](https://github.com/kubernetes-client/'
+                            'python/blob/master/kubernetes/docs/V1ListMeta.md)'),
+
+        lambda l: l.replace('[**V1HTTPGetAction**](V1HTTPGetAction.md)',
+                            '[**V1HTTPGetAction**](https://github.com/kubernetes-client/'
+                            'python/blob/master/kubernetes/docs/V1HTTPGetAction.md)')
+
     ]
 
     src_dirs = [
@@ -48,10 +77,11 @@ def update_python_sdk(src, dest, versions=('v1beta1')):
     ]
 
     for src_dir, dest_dir in zip(src_dirs, dest_dirs):
-        # remove previous generated files explicitly, in case of deprecated instances
+        # Remove previous generated files explicitly, in case of deprecated instances.
         for file in os.listdir(dest_dir):
             path = os.path.join(dest_dir, file)
-            if not os.path.isfile(path):
+            # We should not remove KatibClient doc.
+            if not os.path.isfile(path) or "/docs/KatibClient.md" in path:
                 continue
             for v in versions:
                 if v in file.lower():

diff --git a/hack/gen-python-sdk/swagger_config.json b/hack/gen-python-sdk/swagger_config.json
@@ -1,11 +1,15 @@
 {
-  "packageName" : "katib",
-  "projectName" : "katib",
+  "packageName": "katib",
+  "projectName": "katib",
   "packageVersion": "0.1",
   "importMappings": {
     "V1Container": "from kubernetes.client import V1Container",
     "V1ListMeta": "from kubernetes.client import V1ListMeta",
     "V1ObjectMeta": "from kubernetes.client import V1ObjectMeta",
     "V1HTTPGetAction": "from kubernetes.client import V1HTTPGetAction"
+  },
+  "typeMappings": {
+    "V1Time": "datetime",
+    "V1UnstructuredUnstructured": "dict()"
   }
 }
diff --git a/sdk/python/v1beta1/docs/KatibClient.md b/sdk/python/v1beta1/docs/KatibClient.md
@@ -0,0 +1,157 @@
+# KatibClient
+
+> KatibClient(config_file=None, context=None, client_configuration=None, persist_config=True)
+
+User can load authentication and cluster information from kube-config file and stores them in kubernetes.client.configuration. Parameters are as following:
+
+| Parameter | Description |
+| --- | --- |
+| config_file | Location of kube-config file. Defaults to `~/.kube/config`. Note that the config_file is needed if user want to operate katib SDK in another remote cluster, user must set `config_file` to load kube-config file explicitly, e.g. `KatibClient(config_file="~/.kube/config")`. |
+| context | Set the active context. If is set to None, current_context from config file will be used. |
+| client_configuration | The kubernetes.client.Configuration to set configs to. |
+| persist_config | If True, config file will be updated when changed (e.g GCP token refresh). |
+
+The APIs for KatibClient are as following:
+
+| Class       | Method                                                      | Description                                                               |
+| ----------- | ----------------------------------------------------------- | ------------------------------------------------------------------------- |
+| KatibClient | [create_experiment](#create_experiment)                     | Create Katib Experiment                                                   |
+| KatibClient | [get_experiment](#get_experiment)                           | Get or watch the specified Experiment or all Experiments in the namespace |
+| KatibClient | [delete_experiment](#delete_experiment)                     | Delete specified Experiment                                               |
+| KatibClient | [list_experiments](#list_experiments)                       | List all Experiments with status                                          |
+| KatibClient | [get_experiment_status](#get_experiment_status)             | Get Experiment status                                                     |
+| KatibClient | [is_experiment_succeeded](#is_experiment_succeeded)         | Check if Experiment status is Succeeded                                   |
+| KatibClient | [list_trials](#list_trials)                                 | List all Trials of specified Experiment with status                       |
+| KatibClient | [get_optimal_hyperparameters](#get_optimal_hyperparameters) | Get currentOptimalTrial with parameterAssignments of an Experiment        |
+
+## create_experiment
+
+> create_experiment(experiment, namespace=None)
+
+### Parameters
+
+| Name | Type | Description | Notes |
+| --- | --- | --- | --- |
+| experiment | [V1beta1Experiment](V1beta1Experiment.md) | experiment definition | Required |
+| namespace | str | Namespace for experiment deploying to. If the `namespace` is not defined, will align with experiment definition, or use current or default namespace if namespace is not specified in experiment definition. | Optional |
+
+
+### Return type
+
+object
+
+## get_experiment
+
+> get_experiment(name=None, namespace=None)
+
+Get Experiment in the specified namespace
+
+### Parameters
+
+| Name | Type | Description | Notes |
+| --- | --- | --- | --- |
+| name | str | Experiment name. If the `name` is not specified, will get all experiments in the namespace. | Optional |
+| namespace | str | Experiment's namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+object
+
+## delete_experiment
+
+> delete_experiment(name, namespace=None)
+
+### Parameters
+
+| Name      | Type | Description                                                     | Notes    |
+| --------- | ---- | --------------------------------------------------------------- | -------- |
+| name      | str  | Experiment name.                                                | Required |
+| namespace | str  | Experiment namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+object
+
+## list_experiments
+
+> list_experiments(namespace=None)
+
+List all Experiments with status
+
+### Parameters
+
+| Name      | Type | Description                                                     | Notes    |
+| --------- | ---- | --------------------------------------------------------------- | -------- |
+| namespace | str  | Experiment namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+List
+
+## get_experiment_status
+
+> get_experiment_status(name, namespace=None)
+
+Returns Experiment status, such as Created, Running, Failed, Succeeded or Restarting.
+
+### Parameters
+
+| Name      | Type | Description                                                     | Notes    |
+| --------- | ---- | --------------------------------------------------------------- | -------- |
+| name      | str  | Experiment name.                                                | Required |
+| namespace | str  | Experiment namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+Str
+
+## is_experiment_succeeded
+
+> is_experiment_succeeded(name, namespace=None)
+
+Returns True if Experiment succeeded; false otherwise.
+
+### Parameters
+
+| Name      | Type | Description                                                     | Notes    |
+| --------- | ---- | --------------------------------------------------------------- | -------- |
+| name      | str  | Experiment name.                                                | Required |
+| namespace | str  | Experiment namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+Bool
+
+## list_trials
+
+> list_trials(name, namespace=None)
+
+List all Trials of an Experiment with status
+
+### Parameters
+
+| Name      | Type | Description                                                     | Notes    |
+| --------- | ---- | --------------------------------------------------------------- | -------- |
+| name      | str  | Experiment name.                                                | Required |
+| namespace | str  | Experiment namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+List
+
+## get_optimal_hyperparameters
+
+> get_optimal_hyperparameters(name, namespace=None)
+
+Get currentOptimalTrial with parameterAssignments of an Experiment
+
+### Parameters
+
+| Name      | Type | Description                                                     | Notes    |
+| --------- | ---- | --------------------------------------------------------------- | -------- |
+| name      | str  | Experiment name.                                                | Required |
+| namespace | str  | Experiment namespace. Defaults to current or default namespace. | Optional |
+
+### Return type
+
+Dict
diff --git a/sdk/python/v1beta1/docs/V1beta1CollectorSpec.md b/sdk/python/v1beta1/docs/V1beta1CollectorSpec.md
@@ -3,7 +3,7 @@
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**custom_collector** | [**V1Container**](V1Container.md) | When kind is \&quot;customCollector\&quot;, this field will be used | [optional] 
+**custom_collector** | [**V1Container**](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1Container.md) | When kind is \&quot;customCollector\&quot;, this field will be used | [optional] 
 **kind** | **str** |  | 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

diff --git a/sdk/python/v1beta1/docs/V1beta1Experiment.md b/sdk/python/v1beta1/docs/V1beta1Experiment.md
@@ -5,7 +5,7 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **api_version** | **str** | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources | [optional] 
 **kind** | **str** | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds | [optional] 
-**metadata** | [**V1ObjectMeta**](V1ObjectMeta.md) |  | [optional] 
+**metadata** | [**V1ObjectMeta**](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1ObjectMeta.md) |  | [optional] 
 **spec** | [**V1beta1ExperimentSpec**](V1beta1ExperimentSpec.md) |  | [optional] 
 **status** | [**V1beta1ExperimentStatus**](V1beta1ExperimentStatus.md) |  | [optional] 
 

diff --git a/sdk/python/v1beta1/docs/V1beta1ExperimentCondition.md b/sdk/python/v1beta1/docs/V1beta1ExperimentCondition.md
@@ -3,8 +3,8 @@
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**last_transition_time** | [**V1Time**](V1Time.md) | Last time the condition transitioned from one status to another. | [optional] 
-**last_update_time** | [**V1Time**](V1Time.md) | The last time this condition was updated. | [optional] 
+**last_transition_time** | **datetime** | Last time the condition transitioned from one status to another. | [optional] 
+**last_update_time** | **datetime** | The last time this condition was updated. | [optional] 
 **message** | **str** | A human readable message indicating details about the transition. | [optional] 
 **reason** | **str** | The reason for the condition&#39;s last transition. | [optional] 
 **status** | **str** | Status of the condition, one of True, False, Unknown. | 

diff --git a/sdk/python/v1beta1/docs/V1beta1ExperimentList.md b/sdk/python/v1beta1/docs/V1beta1ExperimentList.md
@@ -6,7 +6,7 @@ Name | Type | Description | Notes
 **api_version** | **str** | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources | [optional] 
 **items** | [**list[V1beta1Experiment]**](V1beta1Experiment.md) |  | 
 **kind** | **str** | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds | [optional] 
-**metadata** | [**V1ListMeta**](V1ListMeta.md) |  | [optional] 
+**metadata** | [**V1ListMeta**](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1ListMeta.md) |  | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

diff --git a/sdk/python/v1beta1/docs/V1beta1ExperimentStatus.md b/sdk/python/v1beta1/docs/V1beta1ExperimentStatus.md
@@ -3,16 +3,16 @@
 ## Properties
 Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
-**completion_time** | [**V1Time**](V1Time.md) | Represents time when the Experiment was completed. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC. | [optional] 
+**completion_time** | **datetime** | Represents time when the Experiment was completed. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC. | [optional] 
 **conditions** | [**list[V1beta1ExperimentCondition]**](V1beta1ExperimentCondition.md) | List of observed runtime conditions for this Experiment. | [optional] 
 **current_optimal_trial** | [**V1beta1OptimalTrial**](V1beta1OptimalTrial.md) | Current optimal trial parameters and observations. | [optional] 
 **early_stopped_trial_list** | **list[str]** | List of trial names which have been early stopped. | [optional] 
 **failed_trial_list** | **list[str]** | List of trial names which have already failed. | [optional] 
 **killed_trial_list** | **list[str]** | List of trial names which have been killed. | [optional] 
-**last_reconcile_time** | [**V1Time**](V1Time.md) | Represents last time when the Experiment was reconciled. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC. | [optional] 
+**last_reconcile_time** | **datetime** | Represents last time when the Experiment was reconciled. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC. | [optional] 
 **pending_trial_list** | **list[str]** | List of trial names which are pending. | [optional] 
 **running_trial_list** | **list[str]** | List of trial names which are running. | [optional] 
-**start_time** | [**V1Time**](V1Time.md) | Represents time when the Experiment was acknowledged by the Experiment controller. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC. | [optional] 
+**start_time** | **datetime** | Represents time when the Experiment was acknowledged by the Experiment controller. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC. | [optional] 
 **succeeded_trial_list** | **list[str]** | List of trial names which have already succeeded. | [optional] 
 **trials** | **int** | Trials is the total number of trials owned by the experiment. | [optional] 
 **trials_early_stopped** | **int** | How many trials are currently early stopped. | [optional] 

diff --git a/sdk/python/v1beta1/docs/V1beta1SourceSpec.md b/sdk/python/v1beta1/docs/V1beta1SourceSpec.md
@@ -5,7 +5,7 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **file_system_path** | [**V1beta1FileSystemPath**](V1beta1FileSystemPath.md) | During training model, metrics may be persisted into local file in source code, such as tfEvent use case | [optional] 
 **filter** | [**V1beta1FilterSpec**](V1beta1FilterSpec.md) | Default metric output format is {\&quot;metric\&quot;: \&quot;&lt;metric_name&gt;\&quot;, \&quot;value\&quot;: &lt;int_or_float&gt;, \&quot;epoch\&quot;: &lt;int&gt;, \&quot;step\&quot;: &lt;int&gt;}, but if the output doesn&#39;t follow default format, please extend it here | [optional] 
-**http_get** | [**V1HTTPGetAction**](V1HTTPGetAction.md) | Model-train source code can expose metrics by http, such as HTTP endpoint in prometheus metric format | [optional] 
+**http_get** | [**V1HTTPGetAction**](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1HTTPGetAction.md) | Model-train source code can expose metrics by http, such as HTTP endpoint in prometheus metric format | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

diff --git a/sdk/python/v1beta1/docs/V1beta1Suggestion.md b/sdk/python/v1beta1/docs/V1beta1Suggestion.md
@@ -5,7 +5,7 @@ Name | Type | Description | Notes
 ------------ | ------------- | ------------- | -------------
 **api_version** | **str** | APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources | [optional] 
 **kind** | **str** | Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds | [optional] 
-**metadata** | [**V1ObjectMeta**](V1ObjectMeta.md) |  | [optional] 
+**metadata** | [**V1ObjectMeta**](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1ObjectMeta.md) |  | [optional] 
 **spec** | [**V1beta1SuggestionSpec**](V1beta1SuggestionSpec.md) |  | [optional] 
 **status** | [**V1beta1SuggestionStatus**](V1beta1SuggestionStatus.md) |  | [optional]