From eb85a8042c7d1d3b477477c5ca8def8934a0929e Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Mon, 28 Dec 2020 10:25:53 -0800 Subject: [PATCH 1/9] fix: issue where GetAvailableLabels endpoint could not be distinguished from GetLabels endpoint --- api/api.swagger.json | 96 +++++++++++++++++++++--------------------- api/gen/label.pb.go | 76 ++++++++++++++++----------------- api/gen/label.pb.gw.go | 2 +- api/proto/label.proto | 2 +- 4 files changed, 88 insertions(+), 88 deletions(-) diff --git a/api/api.swagger.json b/api/api.swagger.json index e1ac9edd..eb60f495 100644 --- a/api/api.swagger.json +++ b/api/api.swagger.json @@ -141,54 +141,6 @@ ] } }, - "/apis/v1beta1/labels/{namespace}/{resource}/labels": { - "get": { - "operationId": "GetAvailableLabels", - "responses": { - "200": { - "description": "A successful response.", - "schema": { - "$ref": "#/definitions/GetLabelsResponse" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/google.rpc.Status" - } - } - }, - "parameters": [ - { - "name": "namespace", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "resource", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "keyLike", - "in": "query", - "required": false, - "type": "string" - }, - { - "name": "skipKeys", - "in": "query", - "required": false, - "type": "string" - } - ], - "tags": [ - "LabelService" - ] - } - }, "/apis/v1beta1/namespaces": { "get": { "operationId": "ListNamespaces", @@ -2758,6 +2710,54 @@ ] } }, + "/apis/v1beta1/{namespace}/{resource}/labels": { + "get": { + "operationId": "GetAvailableLabels", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/GetLabelsResponse" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/google.rpc.Status" + } + } + }, + "parameters": [ + { + "name": "namespace", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "resource", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "keyLike", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "skipKeys", + "in": "query", + "required": false, + "type": "string" + } + ], + "tags": [ + "LabelService" + ] + } + }, "/apis/v1beta1/{namespace}/{resource}/{uid}/labels": { "get": { "operationId": "GetLabels", diff --git a/api/gen/label.pb.go b/api/gen/label.pb.go index ee327a27..da9c54f9 100644 --- a/api/gen/label.pb.go +++ b/api/gen/label.pb.go @@ -577,51 +577,51 @@ var file_label_proto_rawDesc = []byte{ 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x32, 0x98, 0x05, 0x0a, 0x0c, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x12, 0x88, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, + 0x79, 0x32, 0x91, 0x05, 0x0a, 0x0c, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x81, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x3a, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x34, 0x12, 0x32, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x2f, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x75, 0x0a, - 0x09, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x15, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x65, 0x22, 0x33, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2d, 0x12, 0x2b, 0x2f, 0x61, 0x70, 0x69, 0x73, + 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x75, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x12, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x39, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x33, 0x12, 0x31, 0x2f, 0x61, 0x70, 0x69, + 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, + 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x7d, 0x0a, + 0x09, 0x41, 0x64, 0x64, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x15, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x41, 0x64, 0x64, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x39, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x33, 0x12, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x41, 0x82, 0xd3, 0xe4, 0x93, 0x02, + 0x3b, 0x22, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x73, 0x12, 0x7d, 0x0a, 0x09, 0x41, 0x64, 0x64, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x73, 0x12, 0x15, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x41, 0x64, 0x64, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, - 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x41, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3b, 0x22, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, - 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, 0x7b, - 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x3a, 0x06, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x12, 0x85, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x70, 0x6c, - 0x61, 0x63, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x41, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3b, - 0x1a, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, - 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x3a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x7f, 0x0a, 0x0b, 0x44, - 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x17, 0x2e, 0x61, 0x70, 0x69, - 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3f, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x39, 0x2a, 0x37, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, - 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, - 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x2f, 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x42, 0x24, 0x5a, 0x22, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x65, 0x70, 0x61, - 0x6e, 0x65, 0x6c, 0x69, 0x6f, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, - 0x65, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x62, 0x65, 0x6c, 0x73, 0x3a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x85, 0x01, 0x0a, + 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x19, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, 0x70, 0x69, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x41, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3b, 0x1a, 0x31, 0x2f, 0x61, 0x70, 0x69, 0x73, + 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x7d, 0x2f, + 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x3a, 0x06, 0x6c, 0x61, + 0x62, 0x65, 0x6c, 0x73, 0x12, 0x7f, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x12, 0x17, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3f, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x39, 0x2a, 0x37, 0x2f, 0x61, + 0x70, 0x69, 0x73, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x7d, 0x2f, 0x7b, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x7d, 0x2f, 0x7b, 0x75, 0x69, 0x64, 0x7d, 0x2f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x2f, + 0x7b, 0x6b, 0x65, 0x79, 0x7d, 0x42, 0x24, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x6e, 0x65, 0x70, 0x61, 0x6e, 0x65, 0x6c, 0x69, 0x6f, 0x2f, 0x63, + 0x6f, 0x72, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/api/gen/label.pb.gw.go b/api/gen/label.pb.gw.go index 5f49439d..01da4657 100644 --- a/api/gen/label.pb.gw.go +++ b/api/gen/label.pb.gw.go @@ -807,7 +807,7 @@ func RegisterLabelServiceHandlerClient(ctx context.Context, mux *runtime.ServeMu } var ( - pattern_LabelService_GetAvailableLabels_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 1, 0, 4, 1, 5, 3, 1, 0, 4, 1, 5, 4, 2, 2}, []string{"apis", "v1beta1", "labels", "namespace", "resource"}, "")) + pattern_LabelService_GetAvailableLabels_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 1, 0, 4, 1, 5, 3, 2, 4}, []string{"apis", "v1beta1", "namespace", "resource", "labels"}, "")) pattern_LabelService_GetLabels_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 1, 0, 4, 1, 5, 2, 1, 0, 4, 1, 5, 3, 1, 0, 4, 1, 5, 4, 2, 5}, []string{"apis", "v1beta1", "namespace", "resource", "uid", "labels"}, "")) diff --git a/api/proto/label.proto b/api/proto/label.proto index e5c975ac..9297c0ce 100644 --- a/api/proto/label.proto +++ b/api/proto/label.proto @@ -8,7 +8,7 @@ import "google/api/annotations.proto"; service LabelService { rpc GetAvailableLabels (GetAvailableLabelsRequest) returns (GetLabelsResponse) { option (google.api.http) = { - get: "/apis/v1beta1/labels/{namespace}/{resource}/labels" + get: "/apis/v1beta1/{namespace}/{resource}/labels" }; } From 4b6ed47506c0b9e10c07416ca72f291d5b9ffd50 Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 10:09:08 -0800 Subject: [PATCH 2/9] feat: fix issues with comments and auto injection of options for select.nodepool and workflow templates. --- pkg/common_types.go | 5 + pkg/util/extensions/extensions.go | 185 ++++++++++++++++++++++++++++++ pkg/workflow_template.go | 170 +++++++++++++++++++++++---- 3 files changed, 337 insertions(+), 23 deletions(-) create mode 100644 pkg/util/extensions/extensions.go diff --git a/pkg/common_types.go b/pkg/common_types.go index 6496ae8e..49b848ca 100644 --- a/pkg/common_types.go +++ b/pkg/common_types.go @@ -85,6 +85,11 @@ func ParseParametersFromManifest(manifest []byte) ([]Parameter, error) { if parameter.Visibility == nil { parameter.Visibility = ptr.String("public") } + + if parameter.Type == "select.nodepool" { + parameter.Options = make([]*ParameterOption, 0) + parameter.Value = ptr.String("default") + } } if err := IsValidParameters(manifestResult.Arguments.Parameters); err != nil { diff --git a/pkg/util/extensions/extensions.go b/pkg/util/extensions/extensions.go new file mode 100644 index 00000000..08a4de26 --- /dev/null +++ b/pkg/util/extensions/extensions.go @@ -0,0 +1,185 @@ +package extensions + +import ( + "fmt" + "gopkg.in/yaml.v3" + "strings" +) + +type NodePair struct { + Key *yaml.Node + Value *yaml.Node +} + +type YamlIndex struct { + parts []string +} + +// String returns the YamlIndex indicated by the parts separated by "." +// e.g. parent.children.favoriteNumber +func (y *YamlIndex) String() string { + return strings.Join(y.parts, ".") +} + +// CreateYamlIndex creates a YamlIndex that specifies the Key via string parts. +// e.g. a key maybe be: parent.child.favoriteNumber and the returned YamlIndex would reflect this. +func CreateYamlIndex(parts ...string) *YamlIndex { + copyParts := make([]string, len(parts)) + + for i, part := range parts { + copyParts[i] = part + } + + return &YamlIndex{ + parts: copyParts, + } +} + +func HasNode(root *yaml.Node, key *YamlIndex) bool { + if key == nil || len(key.parts) == 0 { + return false + } + + currentNode := root + if len(root.Content) == 1 { + currentNode = root.Content[0] + } + + for _, keyPart := range key.parts { + found := false + for j := 0; j < len(currentNode.Content)-1; j += 2 { + keyNode := currentNode.Content[j] + valueNode := currentNode.Content[j+1] + + if keyNode.Value == keyPart { + currentNode = valueNode + found = true + break + } + } + + if !found { + return false + } + } + + return true +} + +// TODO support indexes +func GetNode(root *yaml.Node, key *YamlIndex) (*yaml.Node, error) { + if key == nil || len(key.parts) == 0 { + return root, nil + } + + currentNode := root + if len(root.Content) == 1 { + currentNode = root.Content[0] + } + + for _, keyPart := range key.parts { + found := false + for j := 0; j < len(currentNode.Content)-1; j += 2 { + keyNode := currentNode.Content[j] + valueNode := currentNode.Content[j+1] + + if keyNode.Value == keyPart { + currentNode = valueNode + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("%v not found - stopped at %v", key.String(), keyPart) + } + } + + return currentNode, nil +} + +func SetKeyValue(node *yaml.Node, key string, value string) error { + if node.Kind != yaml.MappingNode { + return fmt.Errorf("not a mapping node") + } + + for i := 0; i < len(node.Content)-1; i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + + if keyNode.Value == key { + valueNode.Value = value + break + } + } + + return nil +} + +// HasKeyValue checks if the node (assumed to be a mapping node) has a key with the given value. +// If it does not, (false, nil) is returned. If there is an error, like a key not existing, an error is returned. +func HasKeyValue(node *yaml.Node, key string, value string) (bool, error) { + if node.Kind != yaml.MappingNode { + return false, fmt.Errorf("not a mapping node") + } + + for i := 0; i < len(node.Content)-1; i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + + if keyNode.Value == key { + return valueNode.Value == value, nil + } + } + + return false, nil +} + +func Iterate(root *yaml.Node, callable func(parent, value *yaml.Node)) { + for _, child := range root.Content { + callable(root, child) + Iterate(child, callable) + } +} + +func DeleteNode(node *yaml.Node, key *YamlIndex) error { + if node.Kind != yaml.MappingNode { + return fmt.Errorf("not a mapping node") + } + + currentNode := node + for i, keyPart := range key.parts { + found := false + for j := 0; j < len(currentNode.Content)-1; j += 2 { + keyNode := currentNode.Content[j] + valueNode := currentNode.Content[j+1] + + if keyNode.Value == keyPart { + if i != (len(key.parts) - 1) { + currentNode = valueNode + } + found = true + break + } + } + + if !found { + return fmt.Errorf("%v not found - stopped at %v", key.String(), keyPart) + } + } + + keptNodes := make([]*yaml.Node, 0) + finalKey := key.parts[len(key.parts)-1] + for i := 0; i < len(currentNode.Content)-1; i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + + if keyNode.Value != finalKey { + keptNodes = append(keptNodes, keyNode, valueNode) + } + } + + currentNode.Content = keptNodes + + return nil +} diff --git a/pkg/workflow_template.go b/pkg/workflow_template.go index 6ae694ae..8d5dccac 100644 --- a/pkg/workflow_template.go +++ b/pkg/workflow_template.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "github.com/onepanelio/core/pkg/util/extensions" "github.com/onepanelio/core/pkg/util/ptr" "github.com/onepanelio/core/pkg/util/request" pagination "github.com/onepanelio/core/pkg/util/request/pagination" + yaml3 "gopkg.in/yaml.v3" "strconv" "strings" "time" @@ -65,6 +67,87 @@ func (c *Client) replaceSysNodePoolOptions(parameters []Parameter) (result []Par return } +func parameterOptionToNodes(option *ParameterOption) *yaml3.Node { + result := &yaml3.Node{ + Kind: yaml3.MappingNode, + } + + result.Content = append(result.Content, &yaml3.Node{ + Kind: yaml3.ScalarNode, + Value: "name", + }) + + result.Content = append(result.Content, &yaml3.Node{ + Kind: yaml3.ScalarNode, + Value: option.Name, + }) + + result.Content = append(result.Content, &yaml3.Node{ + Kind: yaml3.ScalarNode, + Value: "value", + }) + + result.Content = append(result.Content, &yaml3.Node{ + Kind: yaml3.ScalarNode, + Value: option.Value, + }) + + return result +} + +func parameterOptionsToNodes(options []*ParameterOption) []*yaml3.Node { + result := make([]*yaml3.Node, 0) + + for _, option := range options { + result = append(result, parameterOptionToNodes(option)) + } + + return result +} + +// formatWorkflowTemplateManifest will remove any extraneous values from the workflow template manifest. +// For example, select.nodepool should not have any options. If it does, they are stripped out. +func formatWorkflowTemplateManifest(manifest string) (string, error) { + root := &yaml3.Node{} + err := yaml3.Unmarshal([]byte(manifest), root) + if err != nil { + return "", err + } + + parametersIndex := extensions.CreateYamlIndex("arguments", "parameters") + + if extensions.HasNode(root, parametersIndex) { + resultNode, err := extensions.GetNode(root, parametersIndex) + if err != nil { + return "", err + } + + for _, child := range resultNode.Content { + hasKey, err := extensions.HasKeyValue(child, "type", "select.nodepool") + if err != nil { + return "", err + } + + if hasKey { + if err := extensions.SetKeyValue(child, "value", "default"); err != nil { + return "", err + } + + if err := extensions.DeleteNode(child, extensions.CreateYamlIndex("options")); err != nil { + return "", err + } + } + } + } + + finalManifest, err := yaml3.Marshal(root) + if err != nil { + return "", err + } + + return string(finalManifest), nil +} + func applyWorkflowTemplateFilter(sb sq.SelectBuilder, request *request.Request) (sq.SelectBuilder, error) { if !request.HasFilter() { return sb, nil @@ -201,6 +284,12 @@ func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *Work return nil, nil, err } + newManifest, err := formatWorkflowTemplateManifest(workflowTemplate.Manifest) + if err != nil { + return nil, nil, err + } + workflowTemplate.Manifest = newManifest + params, err := ParseParametersFromManifest([]byte(workflowTemplate.Manifest)) if err != nil { return nil, nil, util.NewUserError(codes.InvalidArgument, err.Error()) @@ -578,6 +667,12 @@ func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplat return nil, fmt.Errorf("uid required for CreateWorkflowTemplateVersion") } + newManifest, err := formatWorkflowTemplateManifest(workflowTemplate.Manifest) + if err != nil { + return nil, err + } + workflowTemplate.Manifest = newManifest + // validate workflow template if err := c.validateWorkflowTemplate(namespace, workflowTemplate); err != nil { return nil, util.NewUserError(codes.InvalidArgument, err.Error()) @@ -1101,36 +1196,65 @@ func (c *Client) GetWorkflowTemplateLabels(namespace, name, prefix string, versi // GenerateWorkflowTemplateManifest replaces any special parameters with runtime values func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, error) { - manifestObject := make(map[string]interface{}) - if err := yaml.Unmarshal([]byte(manifest), &manifestObject); err != nil { - return "", util.NewUserError(codes.InvalidArgument, "Invalid yaml") + root := &yaml3.Node{} + if err := yaml3.Unmarshal([]byte(manifest), root); err != nil { + return "", err } - argumentsRaw := manifestObject["arguments"] - arguments, ok := argumentsRaw.(map[string]interface{}) - if !ok { - return "", fmt.Errorf("unable to parse arguments") - } + parametersIndex := extensions.CreateYamlIndex("arguments", "parameters") - jsonParameters, err := json.Marshal(arguments["parameters"]) - if err != nil { - return "", err - } + if extensions.HasNode(root, parametersIndex) { + resultNode, err := extensions.GetNode(root, extensions.CreateYamlIndex("arguments", "parameters")) + if err != nil { + return "", err + } - parameters := make([]Parameter, 0) - if err := json.Unmarshal(jsonParameters, ¶meters); err != nil { - return "", err - } + nodePoolOptions, err := c.systemConfig.NodePoolOptions() + if err != nil { + return "", err + } - parameters, err = c.replaceSysNodePoolOptions(parameters) - if err != nil { - return "", err - } + nodePoolParameterOptions := make([]*ParameterOption, 0) + for _, option := range nodePoolOptions { + nodePoolParameterOptions = append(nodePoolParameterOptions, &ParameterOption{ + Name: option.Name, + Value: option.Value, + }) + } - arguments["parameters"] = parameters - manifestObject["arguments"] = arguments + for _, child := range resultNode.Content { + hasKey, err := extensions.HasKeyValue(child, "type", "select.nodepool") + if err != nil { + return "", err + } + + if hasKey { + if err := extensions.SetKeyValue(child, "value", nodePoolParameterOptions[0].Value); err != nil { + return "", err + } + + optionsIndex := extensions.CreateYamlIndex("options") + if !extensions.HasNode(child, optionsIndex) { + child.Content = append(child.Content, &yaml3.Node{ + Kind: yaml3.ScalarNode, + Value: "options", + }, &yaml3.Node{ + Kind: yaml3.SequenceNode, + }) + } + + optionsNode, err := extensions.GetNode(child, optionsIndex) + if err != nil { + return "", err + } + + optionsNode.Kind = yaml3.SequenceNode + optionsNode.Content = parameterOptionsToNodes(nodePoolParameterOptions) + } + } + } - finalManifest, err := yaml.Marshal(manifestObject) + finalManifest, err := yaml3.Marshal(root) if err != nil { return "", err } From d4e29684777b18424595484fe5cd91297a62efdc Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 10:37:28 -0800 Subject: [PATCH 3/9] chore: document methods --- pkg/util/extensions/extensions.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/util/extensions/extensions.go b/pkg/util/extensions/extensions.go index 08a4de26..488ee88e 100644 --- a/pkg/util/extensions/extensions.go +++ b/pkg/util/extensions/extensions.go @@ -6,11 +6,13 @@ import ( "strings" ) +// NodePair is a convenience wrapper for two nodes, usually a key/value pair. type NodePair struct { Key *yaml.Node Value *yaml.Node } +// YamlIndex identifies a path in a Yaml Node Tree type YamlIndex struct { parts []string } @@ -35,6 +37,7 @@ func CreateYamlIndex(parts ...string) *YamlIndex { } } +// HasNode returns true if the root node has the key func HasNode(root *yaml.Node, key *YamlIndex) bool { if key == nil || len(key.parts) == 0 { return false @@ -66,6 +69,7 @@ func HasNode(root *yaml.Node, key *YamlIndex) bool { return true } +// GetNode returns the node that contains the content for the key // TODO support indexes func GetNode(root *yaml.Node, key *YamlIndex) (*yaml.Node, error) { if key == nil || len(key.parts) == 0 { @@ -98,6 +102,7 @@ func GetNode(root *yaml.Node, key *YamlIndex) (*yaml.Node, error) { return currentNode, nil } +// SetKeyValue set's the content node's value to value for the indicated key func SetKeyValue(node *yaml.Node, key string, value string) error { if node.Kind != yaml.MappingNode { return fmt.Errorf("not a mapping node") @@ -135,6 +140,7 @@ func HasKeyValue(node *yaml.Node, key string, value string) (bool, error) { return false, nil } +// Iterate runs through all of the content nodes in the indicated root node func Iterate(root *yaml.Node, callable func(parent, value *yaml.Node)) { for _, child := range root.Content { callable(root, child) @@ -142,6 +148,7 @@ func Iterate(root *yaml.Node, callable func(parent, value *yaml.Node)) { } } +// DeleteNode will delete the key and content nodes for the given key func DeleteNode(node *yaml.Node, key *YamlIndex) error { if node.Kind != yaml.MappingNode { return fmt.Errorf("not a mapping node") From 6a4c1e8917c7770fe4b530eb024b9933d6957ef0 Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 10:54:35 -0800 Subject: [PATCH 4/9] chore: code clean up and documentation --- pkg/util/extensions/extensions.go | 1 + pkg/workflow_template.go | 30 ++++++++++-------------------- pkg/workspace_template.go | 9 +-------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/pkg/util/extensions/extensions.go b/pkg/util/extensions/extensions.go index 488ee88e..f1c2f59f 100644 --- a/pkg/util/extensions/extensions.go +++ b/pkg/util/extensions/extensions.go @@ -25,6 +25,7 @@ func (y *YamlIndex) String() string { // CreateYamlIndex creates a YamlIndex that specifies the Key via string parts. // e.g. a key maybe be: parent.child.favoriteNumber and the returned YamlIndex would reflect this. +// Note: this does not yet support indexing array values. func CreateYamlIndex(parts ...string) *YamlIndex { copyParts := make([]string, len(parts)) diff --git a/pkg/workflow_template.go b/pkg/workflow_template.go index 8d5dccac..e4f663fd 100644 --- a/pkg/workflow_template.go +++ b/pkg/workflow_template.go @@ -38,23 +38,15 @@ func (wt *WorkflowTemplateFilter) GetLabels() []*Label { // replaceSysNodePoolOptions replaces a select.nodepool parameter with the nodePool options in the systemConfig // and returns the new parameters with the change. func (c *Client) replaceSysNodePoolOptions(parameters []Parameter) (result []Parameter, err error) { - nodePoolOptions, err := c.systemConfig.NodePoolOptions() + nodePoolOptions, err := c.systemConfig.NodePoolOptionsAsParameters() if err != nil { return result, err } - nodePoolParameterOptions := make([]*ParameterOption, 0) - for _, option := range nodePoolOptions { - nodePoolParameterOptions = append(nodePoolParameterOptions, &ParameterOption{ - Name: option.Name, - Value: option.Value, - }) - } - for i := range parameters { param := parameters[i] if param.Type == "select.nodepool" { - param.Options = nodePoolParameterOptions + param.Options = nodePoolOptions if param.Value != nil && *param.Value == "default" { param.Value = ptr.String(param.Options[0].Value) @@ -67,6 +59,7 @@ func (c *Client) replaceSysNodePoolOptions(parameters []Parameter) (result []Par return } +// parameterOptionToNodes returns a mapping Node where the content's are the options name/value func parameterOptionToNodes(option *ParameterOption) *yaml3.Node { result := &yaml3.Node{ Kind: yaml3.MappingNode, @@ -95,6 +88,7 @@ func parameterOptionToNodes(option *ParameterOption) *yaml3.Node { return result } +// parameterOptionsToNodes returns an array of nodes representing the options func parameterOptionsToNodes(options []*ParameterOption) []*yaml3.Node { result := make([]*yaml3.Node, 0) @@ -1204,22 +1198,18 @@ func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, erro parametersIndex := extensions.CreateYamlIndex("arguments", "parameters") if extensions.HasNode(root, parametersIndex) { - resultNode, err := extensions.GetNode(root, extensions.CreateYamlIndex("arguments", "parameters")) + resultNode, err := extensions.GetNode(root, parametersIndex) if err != nil { return "", err } - nodePoolOptions, err := c.systemConfig.NodePoolOptions() + nodePoolOptions, err := c.systemConfig.NodePoolOptionsAsParameters() if err != nil { return "", err } - nodePoolParameterOptions := make([]*ParameterOption, 0) - for _, option := range nodePoolOptions { - nodePoolParameterOptions = append(nodePoolParameterOptions, &ParameterOption{ - Name: option.Name, - Value: option.Value, - }) + if len(nodePoolOptions) == 0 { + return "", fmt.Errorf("no node pool options") } for _, child := range resultNode.Content { @@ -1229,7 +1219,7 @@ func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, erro } if hasKey { - if err := extensions.SetKeyValue(child, "value", nodePoolParameterOptions[0].Value); err != nil { + if err := extensions.SetKeyValue(child, "value", nodePoolOptions[0].Value); err != nil { return "", err } @@ -1249,7 +1239,7 @@ func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, erro } optionsNode.Kind = yaml3.SequenceNode - optionsNode.Content = parameterOptionsToNodes(nodePoolParameterOptions) + optionsNode.Content = parameterOptionsToNodes(nodePoolOptions) } } } diff --git a/pkg/workspace_template.go b/pkg/workspace_template.go index ba176433..1804964b 100644 --- a/pkg/workspace_template.go +++ b/pkg/workspace_template.go @@ -180,17 +180,10 @@ func generateRuntimeParameters(config SystemConfig) (parameters []Parameter, err }) // Node pool parameter and options - nodePoolOptions, err := config.NodePoolOptions() + options, err := config.NodePoolOptionsAsParameters() if err != nil { return nil, err } - var options []*ParameterOption - for _, option := range nodePoolOptions { - options = append(options, &ParameterOption{ - Name: option.Name, - Value: option.Value, - }) - } if len(options) == 0 { return nil, fmt.Errorf("no node pool options in config") } From 13170cbbeb92fc636b0f6a6cbe89334225aa5225 Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 10:54:47 -0800 Subject: [PATCH 5/9] fix: add missing method addition --- pkg/config_types.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/config_types.go b/pkg/config_types.go index 8f7fb312..58c69b83 100644 --- a/pkg/config_types.go +++ b/pkg/config_types.go @@ -113,6 +113,24 @@ func (s SystemConfig) NodePoolOptions() (options []*NodePoolOption, err error) { return } +// NodePoolOptionsAsParameters returns the NodePool options as []*ParameterOption +func (s SystemConfig) NodePoolOptionsAsParameters() (result []*ParameterOption, err error) { + nodePoolOptions, err := s.NodePoolOptions() + if err != nil { + return nil, err + } + + result = make([]*ParameterOption, 0) + for _, option := range nodePoolOptions { + result = append(result, &ParameterOption{ + Name: option.Name, + Value: option.Value, + }) + } + + return +} + // NodePoolOptionByValue returns the nodePoolOption based on a given value func (s SystemConfig) NodePoolOptionByValue(value string) (option *NodePoolOption, err error) { options, err := s.NodePoolOptions() From ef941de5245d8e4ad4ddea2d30bf7d48eb3df7ba Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 11:14:19 -0800 Subject: [PATCH 6/9] fix: case where options might not exist in manifest --- pkg/workflow_template.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/workflow_template.go b/pkg/workflow_template.go index e4f663fd..7f789c72 100644 --- a/pkg/workflow_template.go +++ b/pkg/workflow_template.go @@ -127,8 +127,11 @@ func formatWorkflowTemplateManifest(manifest string) (string, error) { return "", err } - if err := extensions.DeleteNode(child, extensions.CreateYamlIndex("options")); err != nil { - return "", err + optionsIndex := extensions.CreateYamlIndex("options") + if extensions.HasNode(child, optionsIndex) { + if err := extensions.DeleteNode(child, optionsIndex); err != nil { + return "", err + } } } } From 1af1ccb2b9fe92d448413974c1ca8595e5c34a51 Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 13:06:09 -0800 Subject: [PATCH 7/9] fix: updated logic so that workflow templates are not modified from their display value, but the generated value provides more error handling --- ...201229205644_fix_jupyter_workspace_yaml.go | 28 ++++++ db/go/db.go | 1 + .../workspaces/jupyterlab/20201229205644.yaml | 93 +++++++++++++++++++ pkg/workflow_template.go | 83 ++++------------- 4 files changed, 140 insertions(+), 65 deletions(-) create mode 100644 db/go/20201229205644_fix_jupyter_workspace_yaml.go create mode 100644 db/yaml/workspaces/jupyterlab/20201229205644.yaml diff --git a/db/go/20201229205644_fix_jupyter_workspace_yaml.go b/db/go/20201229205644_fix_jupyter_workspace_yaml.go new file mode 100644 index 00000000..92543f69 --- /dev/null +++ b/db/go/20201229205644_fix_jupyter_workspace_yaml.go @@ -0,0 +1,28 @@ +package migration + +import ( + "database/sql" + "github.com/pressly/goose" + "path/filepath" +) + +func initialize20201229205644() { + if _, ok := initializedMigrations[20201229205644]; !ok { + goose.AddMigration(Up20201229205644, Down20201229205644) + initializedMigrations[20201229205644] = true + } +} + +func Up20201229205644(tx *sql.Tx) error { + // This code is executed when the migration is applied. + return updateWorkspaceTemplateManifest( + filepath.Join("workspaces", "jupyterlab", "20201229205644.yaml"), + jupyterLabTemplateName) +} + +func Down20201229205644(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + return updateWorkspaceTemplateManifest( + filepath.Join("workspaces", "jupyterlab", "20201214133458.yaml"), + jupyterLabTemplateName) +} diff --git a/db/go/db.go b/db/go/db.go index ce8591cc..27ff509a 100644 --- a/db/go/db.go +++ b/db/go/db.go @@ -83,6 +83,7 @@ func Initialize() { initialize20201221195937() initialize20201223062947() initialize20201223202929() + initialize20201229205644() if err := client.DB.Close(); err != nil { log.Printf("[error] closing db %v", err) diff --git a/db/yaml/workspaces/jupyterlab/20201229205644.yaml b/db/yaml/workspaces/jupyterlab/20201229205644.yaml new file mode 100644 index 00000000..521f24f7 --- /dev/null +++ b/db/yaml/workspaces/jupyterlab/20201229205644.yaml @@ -0,0 +1,93 @@ +# Docker containers that are part of the Workspace +containers: + - name: jupyterlab + image: onepanel/dl:0.17.0 + command: ["/bin/bash", "-c", "pip install onepanel-sdk && start.sh LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64 jupyter lab --LabApp.token='' --LabApp.allow_remote_access=True --LabApp.allow_origin=\"*\" --LabApp.disable_check_xsrf=True --LabApp.trust_xheaders=True --LabApp.base_url=/ --LabApp.tornado_settings='{\"headers\":{\"Content-Security-Policy\":\"frame-ancestors * 'self'\"}}' --notebook-dir='/data' --allow-root"] + workingDir: /data + env: + - name: tornado + value: "'{'headers':{'Content-Security-Policy':\"frame-ancestors\ *\ 'self'\"}}'" + - name: TENSORBOARD_PROXY_URL + value: '//$(ONEPANEL_RESOURCE_UID)--$(ONEPANEL_RESOURCE_NAMESPACE).$(ONEPANEL_DOMAIN)/tensorboard' + ports: + - containerPort: 8888 + name: jupyterlab + - containerPort: 6006 + name: tensorboard + - containerPort: 8080 + name: nni + volumeMounts: + - name: data + mountPath: /data + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -c + - > + condayml="/data/.environment.yml"; + jupytertxt="/data/.jupexported.txt"; + if [ -f "$condayml" ]; then conda env update -f $condayml; fi; + if [ -f "$jupytertxt" ]; then cat $jupytertxt | xargs -n 1 jupyter labextension install --no-build && jupyter lab build --minimize=False; fi; + preStop: + exec: + command: + - /bin/sh + - -c + - > + conda env export > /data/.environment.yml -n base; + jupyter labextension list 1>/dev/null 2> /data/.jup.txt; + cat /data/.jup.txt | sed -n '2,$p' | awk 'sub(/v/,"@", $2){print $1$2}' > /data/.jupexported.txt; +ports: + - name: jupyterlab + port: 80 + protocol: TCP + targetPort: 8888 + - name: tensorboard + port: 6006 + protocol: TCP + targetPort: 6006 + - name: nni + port: 8080 + protocol: TCP + targetPort: 8080 +routes: + - match: + - uri: + prefix: /tensorboard + route: + - destination: + port: + number: 6006 + - match: + - uri: + prefix: /nni + route: + - destination: + port: + number: 8080 + - match: + - uri: + prefix: / #jupyter runs at the default route + route: + - destination: + port: + number: 80 +# DAG Workflow to be executed once a Workspace action completes (optional) +#postExecutionWorkflow: +# entrypoint: main +# templates: +# - name: main +# dag: +# tasks: +# - name: slack-notify +# template: slack-notify +# - name: slack-notify +# container: +# image: technosophos/slack-notify +# args: +# - SLACK_USERNAME=onepanel SLACK_TITLE="Your workspace is ready" SLACK_ICON=https://www.gravatar.com/avatar/5c4478592fe00878f62f0027be59c1bd SLACK_MESSAGE="Your workspace is now running" ./slack-notify +# command: +# - sh +# - -c diff --git a/pkg/workflow_template.go b/pkg/workflow_template.go index 7f789c72..dc259038 100644 --- a/pkg/workflow_template.go +++ b/pkg/workflow_template.go @@ -99,52 +99,6 @@ func parameterOptionsToNodes(options []*ParameterOption) []*yaml3.Node { return result } -// formatWorkflowTemplateManifest will remove any extraneous values from the workflow template manifest. -// For example, select.nodepool should not have any options. If it does, they are stripped out. -func formatWorkflowTemplateManifest(manifest string) (string, error) { - root := &yaml3.Node{} - err := yaml3.Unmarshal([]byte(manifest), root) - if err != nil { - return "", err - } - - parametersIndex := extensions.CreateYamlIndex("arguments", "parameters") - - if extensions.HasNode(root, parametersIndex) { - resultNode, err := extensions.GetNode(root, parametersIndex) - if err != nil { - return "", err - } - - for _, child := range resultNode.Content { - hasKey, err := extensions.HasKeyValue(child, "type", "select.nodepool") - if err != nil { - return "", err - } - - if hasKey { - if err := extensions.SetKeyValue(child, "value", "default"); err != nil { - return "", err - } - - optionsIndex := extensions.CreateYamlIndex("options") - if extensions.HasNode(child, optionsIndex) { - if err := extensions.DeleteNode(child, optionsIndex); err != nil { - return "", err - } - } - } - } - } - - finalManifest, err := yaml3.Marshal(root) - if err != nil { - return "", err - } - - return string(finalManifest), nil -} - func applyWorkflowTemplateFilter(sb sq.SelectBuilder, request *request.Request) (sq.SelectBuilder, error) { if !request.HasFilter() { return sb, nil @@ -281,12 +235,6 @@ func (c *Client) createWorkflowTemplate(namespace string, workflowTemplate *Work return nil, nil, err } - newManifest, err := formatWorkflowTemplateManifest(workflowTemplate.Manifest) - if err != nil { - return nil, nil, err - } - workflowTemplate.Manifest = newManifest - params, err := ParseParametersFromManifest([]byte(workflowTemplate.Manifest)) if err != nil { return nil, nil, util.NewUserError(codes.InvalidArgument, err.Error()) @@ -664,12 +612,6 @@ func (c *Client) CreateWorkflowTemplateVersion(namespace string, workflowTemplat return nil, fmt.Errorf("uid required for CreateWorkflowTemplateVersion") } - newManifest, err := formatWorkflowTemplateManifest(workflowTemplate.Manifest) - if err != nil { - return nil, err - } - workflowTemplate.Manifest = newManifest - // validate workflow template if err := c.validateWorkflowTemplate(namespace, workflowTemplate); err != nil { return nil, util.NewUserError(codes.InvalidArgument, err.Error()) @@ -1222,20 +1164,31 @@ func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, erro } if hasKey { + isDefault, err := extensions.HasKeyValue(child, "value", "default") + if err != nil { + return "", err + } + if !isDefault { + return "", util.NewUserError(codes.InvalidArgument, "select.nodepool must have a value of 'default'") + } + if err := extensions.SetKeyValue(child, "value", nodePoolOptions[0].Value); err != nil { return "", err } optionsIndex := extensions.CreateYamlIndex("options") - if !extensions.HasNode(child, optionsIndex) { - child.Content = append(child.Content, &yaml3.Node{ - Kind: yaml3.ScalarNode, - Value: "options", - }, &yaml3.Node{ - Kind: yaml3.SequenceNode, - }) + + if extensions.HasNode(child, optionsIndex) { + return "", util.NewUserError(codes.InvalidArgument, "select.nodepool must not have any options") } + child.Content = append(child.Content, &yaml3.Node{ + Kind: yaml3.ScalarNode, + Value: "options", + }, &yaml3.Node{ + Kind: yaml3.SequenceNode, + }) + optionsNode, err := extensions.GetNode(child, optionsIndex) if err != nil { return "", err From 01f3fc664ba92e7f5aad04dfd129eb2e4f093c42 Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 13:18:00 -0800 Subject: [PATCH 8/9] fix: updated generated workflow template to check all valid node pool values --- pkg/util/extensions/extensions.go | 10 +++++++--- pkg/workflow_template.go | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/util/extensions/extensions.go b/pkg/util/extensions/extensions.go index f1c2f59f..8d28c8ac 100644 --- a/pkg/util/extensions/extensions.go +++ b/pkg/util/extensions/extensions.go @@ -122,9 +122,9 @@ func SetKeyValue(node *yaml.Node, key string, value string) error { return nil } -// HasKeyValue checks if the node (assumed to be a mapping node) has a key with the given value. +// HasKeyValue checks if the node (assumed to be a mapping node) has a key with the given value(s). If many values, any is ok. // If it does not, (false, nil) is returned. If there is an error, like a key not existing, an error is returned. -func HasKeyValue(node *yaml.Node, key string, value string) (bool, error) { +func HasKeyValue(node *yaml.Node, key string, values ...string) (bool, error) { if node.Kind != yaml.MappingNode { return false, fmt.Errorf("not a mapping node") } @@ -134,7 +134,11 @@ func HasKeyValue(node *yaml.Node, key string, value string) (bool, error) { valueNode := node.Content[i+1] if keyNode.Value == key { - return valueNode.Value == value, nil + for _, val := range values { + if valueNode.Value == val { + return true, nil + } + } } } diff --git a/pkg/workflow_template.go b/pkg/workflow_template.go index dc259038..d7dab49a 100644 --- a/pkg/workflow_template.go +++ b/pkg/workflow_template.go @@ -1152,6 +1152,10 @@ func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, erro if err != nil { return "", err } + legalNodePoolValues := make([]string, 0) + for _, opt := range nodePoolOptions { + legalNodePoolValues = append(legalNodePoolValues, opt.Value) + } if len(nodePoolOptions) == 0 { return "", fmt.Errorf("no node pool options") @@ -1164,12 +1168,12 @@ func (c *Client) GenerateWorkflowTemplateManifest(manifest string) (string, erro } if hasKey { - isDefault, err := extensions.HasKeyValue(child, "value", "default") + isValid, err := extensions.HasKeyValue(child, "value", legalNodePoolValues...) if err != nil { return "", err } - if !isDefault { - return "", util.NewUserError(codes.InvalidArgument, "select.nodepool must have a value of 'default'") + if !isValid { + return "", util.NewUserError(codes.InvalidArgument, fmt.Sprintf("select.nodepool has an invalid value. Valid values are: %v", strings.Join(legalNodePoolValues, ", "))) } if err := extensions.SetKeyValue(child, "value", nodePoolOptions[0].Value); err != nil { From 61c31e43f77aaf9d71822879caf218563ebde81a Mon Sep 17 00:00:00 2001 From: Andrey Melnikov Date: Tue, 29 Dec 2020 13:26:18 -0800 Subject: [PATCH 9/9] chore: migration method documentation --- db/go/20201229205644_fix_jupyter_workspace_yaml.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/go/20201229205644_fix_jupyter_workspace_yaml.go b/db/go/20201229205644_fix_jupyter_workspace_yaml.go index 92543f69..389dee55 100644 --- a/db/go/20201229205644_fix_jupyter_workspace_yaml.go +++ b/db/go/20201229205644_fix_jupyter_workspace_yaml.go @@ -13,6 +13,7 @@ func initialize20201229205644() { } } +// Up20201229205644 updates the jupyterlab workspace template func Up20201229205644(tx *sql.Tx) error { // This code is executed when the migration is applied. return updateWorkspaceTemplateManifest( @@ -20,6 +21,7 @@ func Up20201229205644(tx *sql.Tx) error { jupyterLabTemplateName) } +// Down20201229205644 rolls back the jupyterab workspace template update func Down20201229205644(tx *sql.Tx) error { // This code is executed when the migration is rolled back. return updateWorkspaceTemplateManifest(