From a0cbfe2a3875a243467df9716fe975059f63c147 Mon Sep 17 00:00:00 2001 From: YundongYe Date: Wed, 18 Dec 2019 20:38:41 +0800 Subject: [PATCH 01/17] init commit --- .../cluster_object_model.py | 9 +++++++- src/alert-manager/config/alert-manager.yaml | 21 ++++++++++++++++++- src/cleaner/config/cleaner.yaml | 2 ++ src/cluster/config/cluster.yaml | 1 + src/device-plugin/config/device-plugin.yaml | 2 ++ src/drivers/config/drivers.yaml | 1 + src/elasticsearch/config/elasticsearch.yaml | 18 ++++++++++++++++ src/grafana/config/grafana.yaml | 19 +++++++++++++++++ .../config/hadoop-batch-job.yaml | 1 + .../config/hadoop-data-node.yaml | 2 ++ .../config/hadoop-jobhistory.yaml | 3 +++ .../config/hadoop-name-node.yaml | 2 ++ .../config/hadoop-node-manager.yaml | 2 ++ .../config/hadoop-resource-manager.yaml | 2 ++ src/hivedscheduler/config/hivedscheduler.yaml | 20 ++++++++++++++++++ .../config/internal-storage.yaml | 19 +++++++++++++++++ src/job-exit-spec/config/job-exit-spec.yaml | 2 ++ 17 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 src/elasticsearch/config/elasticsearch.yaml create mode 100644 src/hivedscheduler/config/hivedscheduler.yaml diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index f4c092316d..13ca00b379 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -60,14 +60,21 @@ def get_service_parser(self, service_name, cluster_type): # Prepare Service Configuration layout = self.layout - + parser_type = "common" default_service_cfg = [] if file_handler.file_exist_or_not(default_path): default_service_cfg = file_handler.load_yaml_config(default_path) + if default_service_cfg is not None and "openpai_parser_type" in default_service_cfg: + parser_type = default_service_cfg["openpai_parser_type"] overwrite_service_cfg = {} if self.overwrite_service_configuration is not None and service_name in self.overwrite_service_configuration: overwrite_service_cfg = self.overwrite_service_configuration[service_name] + if "openpai_parser_type" in overwrite_service_cfg: + parser_type = overwrite_service_cfg["openpai_parser_type"] + + if parser_type != "common" and parser_type != cluster_type: + return None # Init parser instance parser_module = importlib.import_module(service_name.replace("-", "_")) diff --git a/src/alert-manager/config/alert-manager.yaml b/src/alert-manager/config/alert-manager.yaml index bb4b426b6a..2d4ac2eaff 100644 --- a/src/alert-manager/config/alert-manager.yaml +++ b/src/alert-manager/config/alert-manager.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" + port: 9093 use-pylon: False -repeat-interval: '24h' +repeat-interval: '24h' \ No newline at end of file diff --git a/src/cleaner/config/cleaner.yaml b/src/cleaner/config/cleaner.yaml index bd053d9cfe..b484b40043 100644 --- a/src/cleaner/config/cleaner.yaml +++ b/src/cleaner/config/cleaner.yaml @@ -15,5 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + threshold: 90 interval: 60 \ No newline at end of file diff --git a/src/cluster/config/cluster.yaml b/src/cluster/config/cluster.yaml index 29e4cf94a3..34fc3b64ae 100644 --- a/src/cluster/config/cluster.yaml +++ b/src/cluster/config/cluster.yaml @@ -15,6 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "common" common: cluster-id: pai diff --git a/src/device-plugin/config/device-plugin.yaml b/src/device-plugin/config/device-plugin.yaml index 7c89e66054..5654a9bf39 100644 --- a/src/device-plugin/config/device-plugin.yaml +++ b/src/device-plugin/config/device-plugin.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "k8s" + devices: - "nvidia.com/gpu" - "rdma/hca" diff --git a/src/drivers/config/drivers.yaml b/src/drivers/config/drivers.yaml index c538f1d8c5..03916af118 100644 --- a/src/drivers/config/drivers.yaml +++ b/src/drivers/config/drivers.yaml @@ -15,6 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" set-nvidia-runtime: false diff --git a/src/elasticsearch/config/elasticsearch.yaml b/src/elasticsearch/config/elasticsearch.yaml new file mode 100644 index 0000000000..2dab086914 --- /dev/null +++ b/src/elasticsearch/config/elasticsearch.yaml @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "k8s" diff --git a/src/grafana/config/grafana.yaml b/src/grafana/config/grafana.yaml index 87c3f5a1c1..1509ac1430 100644 --- a/src/grafana/config/grafana.yaml +++ b/src/grafana/config/grafana.yaml @@ -1 +1,20 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" + port: 3000 diff --git a/src/hadoop-batch-job/config/hadoop-batch-job.yaml b/src/hadoop-batch-job/config/hadoop-batch-job.yaml index dbde8d9a14..a3deedd44d 100644 --- a/src/hadoop-batch-job/config/hadoop-batch-job.yaml +++ b/src/hadoop-batch-job/config/hadoop-batch-job.yaml @@ -15,3 +15,4 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" diff --git a/src/hadoop-data-node/config/hadoop-data-node.yaml b/src/hadoop-data-node/config/hadoop-data-node.yaml index 0be1c28332..e6a636b397 100644 --- a/src/hadoop-data-node/config/hadoop-data-node.yaml +++ b/src/hadoop-data-node/config/hadoop-data-node.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + storage_path: diff --git a/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml b/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml index dbde8d9a14..17babcd520 100644 --- a/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml +++ b/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml @@ -15,3 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + + diff --git a/src/hadoop-name-node/config/hadoop-name-node.yaml b/src/hadoop-name-node/config/hadoop-name-node.yaml index dbde8d9a14..ac08d35cea 100644 --- a/src/hadoop-name-node/config/hadoop-name-node.yaml +++ b/src/hadoop-name-node/config/hadoop-name-node.yaml @@ -15,3 +15,5 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + diff --git a/src/hadoop-node-manager/config/hadoop-node-manager.yaml b/src/hadoop-node-manager/config/hadoop-node-manager.yaml index dbde8d9a14..ac08d35cea 100644 --- a/src/hadoop-node-manager/config/hadoop-node-manager.yaml +++ b/src/hadoop-node-manager/config/hadoop-node-manager.yaml @@ -15,3 +15,5 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml index 2330e758e1..0eabd91f49 100644 --- a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml +++ b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + virtualClusters: default: description: Default VC. diff --git a/src/hivedscheduler/config/hivedscheduler.yaml b/src/hivedscheduler/config/hivedscheduler.yaml new file mode 100644 index 0000000000..17babcd520 --- /dev/null +++ b/src/hivedscheduler/config/hivedscheduler.yaml @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "yarn" + + diff --git a/src/internal-storage/config/internal-storage.yaml b/src/internal-storage/config/internal-storage.yaml index 4c6baf2d70..ef5f39d5d5 100644 --- a/src/internal-storage/config/internal-storage.yaml +++ b/src/internal-storage/config/internal-storage.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" + enable: false type: hostPath root-path: /mnt/paiInternal diff --git a/src/job-exit-spec/config/job-exit-spec.yaml b/src/job-exit-spec/config/job-exit-spec.yaml index e9d6dbaad6..d1662b2dcd 100644 --- a/src/job-exit-spec/config/job-exit-spec.yaml +++ b/src/job-exit-spec/config/job-exit-spec.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + ################################################################################ # PAI Job ExitSpec Schema ################################################################################ From f7627f7ddc865207d8f17c36702c3ca6ee10feee Mon Sep 17 00:00:00 2001 From: YundongYe Date: Wed, 18 Dec 2019 21:58:19 +0800 Subject: [PATCH 02/17] init commit --- src/job-exporter/config/job-exporter.yaml | 32 ++++++++++--------- src/k8s-dashboard/config/k8s-dashboard.yaml | 2 ++ .../config/k8s-job-exit-spec.yaml | 2 ++ src/log-manager/config/log-manager.yaml | 2 ++ src/node-exporter/config/node-exporter.yaml | 32 ++++++++++--------- src/postgresql/config/postgresql.yaml | 19 +++++++++++ src/prometheus/config/prometheus.yaml | 19 +++++++++++ src/pylon/config/pylon.yaml | 19 +++++++++++ src/rest-server/config/rest-server.k8s.yaml | 2 ++ src/rest-server/config/rest-server.yaml | 2 ++ .../config/storage-manager.yaml | 2 ++ src/webportal/config/webportal.k8s.yaml | 4 ++- src/webportal/config/webportal.yaml | 4 ++- .../config/yarn-frameworklauncher.yaml | 2 ++ src/zookeeper/config/zookeeper.yaml | 2 ++ 15 files changed, 113 insertions(+), 32 deletions(-) diff --git a/src/job-exporter/config/job-exporter.yaml b/src/job-exporter/config/job-exporter.yaml index e895fafc00..f1589f2cd0 100644 --- a/src/job-exporter/config/job-exporter.yaml +++ b/src/job-exporter/config/job-exporter.yaml @@ -1,19 +1,21 @@ # Copyright (c) Microsoft Corporation -# # All rights reserved. -# # -# # MIT License -# # -# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# # documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# # to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# # -# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" port: 9102 logging-level: INFO diff --git a/src/k8s-dashboard/config/k8s-dashboard.yaml b/src/k8s-dashboard/config/k8s-dashboard.yaml index afedca73fa..70bd417407 100644 --- a/src/k8s-dashboard/config/k8s-dashboard.yaml +++ b/src/k8s-dashboard/config/k8s-dashboard.yaml @@ -14,3 +14,5 @@ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" \ No newline at end of file diff --git a/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml b/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml index 43858e9a6e..9dc3c7d6ae 100644 --- a/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml +++ b/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "k8s" + ################################################################################ # PAI Job ExitSpec Schema ################################################################################ diff --git a/src/log-manager/config/log-manager.yaml b/src/log-manager/config/log-manager.yaml index ccfa273b9e..cc015283d9 100644 --- a/src/log-manager/config/log-manager.yaml +++ b/src/log-manager/config/log-manager.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "k8s" + port: 9103 diff --git a/src/node-exporter/config/node-exporter.yaml b/src/node-exporter/config/node-exporter.yaml index e5f0bf30b7..bb3fd558c9 100644 --- a/src/node-exporter/config/node-exporter.yaml +++ b/src/node-exporter/config/node-exporter.yaml @@ -1,18 +1,20 @@ # Copyright (c) Microsoft Corporation -# # All rights reserved. -# # -# # MIT License -# # -# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# # documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# # to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# # -# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" port: 9100 diff --git a/src/postgresql/config/postgresql.yaml b/src/postgresql/config/postgresql.yaml index edcb1a503e..8e136f082b 100644 --- a/src/postgresql/config/postgresql.yaml +++ b/src/postgresql/config/postgresql.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" + enable: false user: root passwd: rootpass diff --git a/src/prometheus/config/prometheus.yaml b/src/prometheus/config/prometheus.yaml index 24b45f0a50..cd8acce469 100644 --- a/src/prometheus/config/prometheus.yaml +++ b/src/prometheus/config/prometheus.yaml @@ -1,2 +1,21 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" + port: 9091 scrape_interval: 30 diff --git a/src/pylon/config/pylon.yaml b/src/pylon/config/pylon.yaml index 0278928e96..a4434e2753 100644 --- a/src/pylon/config/pylon.yaml +++ b/src/pylon/config/pylon.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +openpai_parser_type: "common" + port: 80 # The following 2 config is for internal use webhdfs-legacy-port: 50070 diff --git a/src/rest-server/config/rest-server.k8s.yaml b/src/rest-server/config/rest-server.k8s.yaml index c91461e3b1..f0eec402ff 100644 --- a/src/rest-server/config/rest-server.k8s.yaml +++ b/src/rest-server/config/rest-server.k8s.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "k8s" + server-port: 9186 launcher-type: k8s jwt-secret: pai-secret diff --git a/src/rest-server/config/rest-server.yaml b/src/rest-server/config/rest-server.yaml index 2cd616b233..f70c40181f 100644 --- a/src/rest-server/config/rest-server.yaml +++ b/src/rest-server/config/rest-server.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + server-port: 9186 launcher-type: yarn jwt-secret: pai-secret diff --git a/src/storage-manager/config/storage-manager.yaml b/src/storage-manager/config/storage-manager.yaml index 3037bee3d0..8e05dfd8f1 100644 --- a/src/storage-manager/config/storage-manager.yaml +++ b/src/storage-manager/config/storage-manager.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "common" + # Global config localpath: /share diff --git a/src/webportal/config/webportal.k8s.yaml b/src/webportal/config/webportal.k8s.yaml index b2be8e1997..dfcb3de20a 100644 --- a/src/webportal/config/webportal.k8s.yaml +++ b/src/webportal/config/webportal.k8s.yaml @@ -15,5 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "k8s" + server-port: 9286 -log-type: log-manager \ No newline at end of file +log-type: log-manager diff --git a/src/webportal/config/webportal.yaml b/src/webportal/config/webportal.yaml index 5cfc657d00..983942d0c3 100644 --- a/src/webportal/config/webportal.yaml +++ b/src/webportal/config/webportal.yaml @@ -15,5 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + server-port: 9286 -log-type: yarn \ No newline at end of file +log-type: yarn diff --git a/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml b/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml index 2348238d2e..8607bb2175 100644 --- a/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml +++ b/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + frameworklauncher-port: 9086 \ No newline at end of file diff --git a/src/zookeeper/config/zookeeper.yaml b/src/zookeeper/config/zookeeper.yaml index 081aeaac6b..17babcd520 100644 --- a/src/zookeeper/config/zookeeper.yaml +++ b/src/zookeeper/config/zookeeper.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +openpai_parser_type: "yarn" + From 0246c0f504f7d4e6e54f7484ce376711e007bf01 Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 19 Dec 2019 11:37:53 +0800 Subject: [PATCH 03/17] Issue fix. --- deployment/clusterObjectModel/cluster_object_model.py | 3 +++ deployment/clusterObjectModel/test/test_template_generate.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index 13ca00b379..33f2706548 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -135,7 +135,10 @@ def service_config(self): service_model_list = self.get_service_model_list() for service_name in service_model_list: + if service_name is None: + continue parser_dict[service_name] = self.get_service_parser(service_name, cluster_type) + return self.load_config(parser_dict) def kubernetes_config(self): diff --git a/deployment/clusterObjectModel/test/test_template_generate.py b/deployment/clusterObjectModel/test/test_template_generate.py index 130c62fc27..032e9af9fe 100644 --- a/deployment/clusterObjectModel/test/test_template_generate.py +++ b/deployment/clusterObjectModel/test/test_template_generate.py @@ -69,7 +69,7 @@ def test_template_generate(self): for subdir in subdir_list: service_deploy_dir = "{0}/{1}/deploy".format(src_path, subdir) - service_deploy_conf_path = "{0}/{1}/deploy/service.yaml".format(src_path, subdir) + service_deploy_conf_path = "{0}/{1}/deploy/service.yaml".format(src_path, subdir) if file_handler.directory_exits(service_deploy_dir) and file_handler.file_exist_or_not(service_deploy_conf_path): service_list.append(subdir) From 428a98138c6330586c35da2bd7f15ccd3b87e21c Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 19 Dec 2019 11:55:39 +0800 Subject: [PATCH 04/17] Issue fix. --- deployment/clusterObjectModel/cluster_object_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index 33f2706548..8f217e840f 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -135,9 +135,10 @@ def service_config(self): service_model_list = self.get_service_model_list() for service_name in service_model_list: - if service_name is None: + parser = self.get_service_parser(service_name, cluster_type) + if parser is None: continue - parser_dict[service_name] = self.get_service_parser(service_name, cluster_type) + parser_dict[service_name] = parser return self.load_config(parser_dict) From 88a806f69df309a95918cd568b560dd377badeb9 Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 19 Dec 2019 15:07:08 +0800 Subject: [PATCH 05/17] Fix UT --- deployment/clusterObjectModel/test/test_template_generate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployment/clusterObjectModel/test/test_template_generate.py b/deployment/clusterObjectModel/test/test_template_generate.py index 032e9af9fe..a4cef5e817 100644 --- a/deployment/clusterObjectModel/test/test_template_generate.py +++ b/deployment/clusterObjectModel/test/test_template_generate.py @@ -67,11 +67,12 @@ def test_template_generate(self): subdir_list = directory_handler.get_subdirectory_list(src_path) for subdir in subdir_list: - service_deploy_dir = "{0}/{1}/deploy".format(src_path, subdir) service_deploy_conf_path = "{0}/{1}/deploy/service.yaml".format(src_path, subdir) if file_handler.directory_exits(service_deploy_dir) and file_handler.file_exist_or_not(service_deploy_conf_path): - service_list.append(subdir) + service_conf = file_handler.load_yaml_config(service_deploy_conf_path) + if ("cluster-type" not in service_conf) or ("cluster-type" in service_conf and "yarn" in service_conf["cluster-type"]): + service_list.append(subdir) for serv in service_list: service_conf = file_handler.load_yaml_config("{0}/{1}/deploy/service.yaml".format(src_path, serv)) From b935e625cf6d3f8cefe17f86a6a199bf5d7bab1d Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 19 Dec 2019 15:18:56 +0800 Subject: [PATCH 06/17] Fix UT --- deployment/clusterObjectModel/test/test_template_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/clusterObjectModel/test/test_template_generate.py b/deployment/clusterObjectModel/test/test_template_generate.py index a4cef5e817..acceb91fab 100644 --- a/deployment/clusterObjectModel/test/test_template_generate.py +++ b/deployment/clusterObjectModel/test/test_template_generate.py @@ -73,7 +73,7 @@ def test_template_generate(self): service_conf = file_handler.load_yaml_config(service_deploy_conf_path) if ("cluster-type" not in service_conf) or ("cluster-type" in service_conf and "yarn" in service_conf["cluster-type"]): service_list.append(subdir) - + print(service_list) for serv in service_list: service_conf = file_handler.load_yaml_config("{0}/{1}/deploy/service.yaml".format(src_path, serv)) service_template_generater = service_template_generate(com, serv, service_conf) From d9f383243874ddfda1783f9c066dd6acba368f0a Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 19 Dec 2019 16:14:13 +0800 Subject: [PATCH 07/17] Fix UT --- deployment/clusterObjectModel/test/test_template_generate.py | 2 +- src/rest-server/deploy/rest-server.yaml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/clusterObjectModel/test/test_template_generate.py b/deployment/clusterObjectModel/test/test_template_generate.py index acceb91fab..a4cef5e817 100644 --- a/deployment/clusterObjectModel/test/test_template_generate.py +++ b/deployment/clusterObjectModel/test/test_template_generate.py @@ -73,7 +73,7 @@ def test_template_generate(self): service_conf = file_handler.load_yaml_config(service_deploy_conf_path) if ("cluster-type" not in service_conf) or ("cluster-type" in service_conf and "yarn" in service_conf["cluster-type"]): service_list.append(subdir) - print(service_list) + for serv in service_list: service_conf = file_handler.load_yaml_config("{0}/{1}/deploy/service.yaml".format(src_path, serv)) service_template_generater = service_template_generate(com, serv, service_conf) diff --git a/src/rest-server/deploy/rest-server.yaml.template b/src/rest-server/deploy/rest-server.yaml.template index bb72183509..b75b151e5c 100644 --- a/src/rest-server/deploy/rest-server.yaml.template +++ b/src/rest-server/deploy/rest-server.yaml.template @@ -96,7 +96,7 @@ spec: - name: AUTHN_METHOD value: OIDC {% endif %} -{%- if cluster_cfg['elasticsearch']['master-ip'] %} +{%- if 'elasticsearch' in cluster_cfg and cluster_cfg['elasticsearch']['master-ip'] %} - name: ELASTICSEARCH_URI value: http://{{ cluster_cfg['elasticsearch']['master-ip'] }}:30100 {%- endif %} From 0eee594a65b0e5d8ae1e2c430580d3be52562fd1 Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 19 Dec 2019 16:28:56 +0800 Subject: [PATCH 08/17] Fix UT --- src/rest-server/deploy/configmap-create.sh.template | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rest-server/deploy/configmap-create.sh.template b/src/rest-server/deploy/configmap-create.sh.template index a494b84108..73c279725b 100644 --- a/src/rest-server/deploy/configmap-create.sh.template +++ b/src/rest-server/deploy/configmap-create.sh.template @@ -25,8 +25,10 @@ kubectl create configmap auth-configuration --from-file=auth-configmap/ --dry-r kubectl create configmap group-configuration --from-file=group-configmap/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? +{% if cluster_cfg['rest-server']['launcher-type'] == "yarn" %} kubectl create configmap job-exit-spec-configuration --from-file=job-exit-spec-config/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? - +{% else %} kubectl create configmap k8s-job-exit-spec-configuration --from-file=k8s-job-exit-spec-config/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? +{% endif %} popd > /dev/null From 2cd0b95f8df24cd0c0acaf03f70d0dd87a760bd7 Mon Sep 17 00:00:00 2001 From: YundongYe <8675883+ydye@users.noreply.github.com> Date: Mon, 23 Dec 2019 13:38:20 +0800 Subject: [PATCH 09/17] [Cluster Object Model] Check laucher-type and log-type based on cluster-type. (#4038) * Remove .k8s.yaml logic. And remove launcher-type & log-type --- .../cluster_object_model.py | 4 +-- .../services-configuration.yaml.template | 5 ++-- .../deploy/job-exporter.yaml.template | 2 +- src/rest-server/config/rest-server.k8s.yaml | 27 ------------------- src/rest-server/config/rest-server.md | 8 ------ src/rest-server/config/rest-server.yaml | 3 +-- src/rest-server/config/rest_server.py | 3 --- .../deploy/configmap-create.sh.template | 2 +- .../k8s-job-exit-spec.yaml.template | 11 ++++---- .../deploy/rest-server.yaml.template | 8 +++--- src/webportal/config/webportal.k8s.yaml | 21 --------------- src/webportal/config/webportal.md | 8 ------ src/webportal/config/webportal.py | 4 --- src/webportal/config/webportal.yaml | 3 +-- src/webportal/deploy/webportal.yaml.template | 8 ++++-- 15 files changed, 23 insertions(+), 94 deletions(-) delete mode 100644 src/rest-server/config/rest-server.k8s.yaml delete mode 100644 src/webportal/config/webportal.k8s.yaml diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index 8f217e840f..c7d058d10d 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -54,9 +54,7 @@ def get_service_model_list(self): def get_service_parser(self, service_name, cluster_type): sys.path.insert(0, '{0}/../../src/{1}/config'.format(package_directory_com, service_name)) - default_path = "{0}/../../src/{1}/config/{1}.{2}.yaml".format(package_directory_com, service_name, cluster_type) - if not file_handler.file_exist_or_not(default_path): - default_path = "{0}/../../src/{1}/config/{1}.yaml".format(package_directory_com, service_name) + default_path = "{0}/../../src/{1}/config/{1}.yaml".format(package_directory_com, service_name) # Prepare Service Configuration layout = self.layout diff --git a/deployment/quick-start/services-configuration.yaml.template b/deployment/quick-start/services-configuration.yaml.template index 562f33f281..2af5fc86b4 100644 --- a/deployment/quick-start/services-configuration.yaml.template +++ b/deployment/quick-start/services-configuration.yaml.template @@ -65,8 +65,6 @@ cluster: rest-server: -# # launcher type. k8s or yarn -# launcher-type: k8s # database admin username default-pai-admin-username: admin # database admin password @@ -76,12 +74,13 @@ rest-server: # uncomment following section if you want to customize the port of web portal # webportal: # server-port: 9286 -# log-type: yarn + # uncomment following if you want to change customeize grafana # grafana: # port: 3000 + # uncomment following if you want to change customeize drivers #drivers: # set-nvidia-runtime: false diff --git a/src/job-exporter/deploy/job-exporter.yaml.template b/src/job-exporter/deploy/job-exporter.yaml.template index a8a16cc79d..a685c62f59 100644 --- a/src/job-exporter/deploy/job-exporter.yaml.template +++ b/src/job-exporter/deploy/job-exporter.yaml.template @@ -71,7 +71,7 @@ spec: - name: NVIDIA_VISIBLE_DEVICES value: all - name: LAUNCHER_TYPE - value: {{ cluster_cfg['rest-server']['launcher-type'] }} + value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }} {%- if cluster_cfg['cluster']['common']['deploy-in-aks'] == "true" %} - name: DEPLOY_ENV value: aks diff --git a/src/rest-server/config/rest-server.k8s.yaml b/src/rest-server/config/rest-server.k8s.yaml deleted file mode 100644 index f0eec402ff..0000000000 --- a/src/rest-server/config/rest-server.k8s.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) Microsoft Corporation -# All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -openpai_parser_type: "k8s" - -server-port: 9186 -launcher-type: k8s -jwt-secret: pai-secret -jwt-expire-time: '7d' -github-owner: Microsoft -github-repository: pai -github-path: marketplace -debugging-reservation-seconds: 604800 diff --git a/src/rest-server/config/rest-server.md b/src/rest-server/config/rest-server.md index 48cd8fce4d..79d5892272 100644 --- a/src/rest-server/config/rest-server.md +++ b/src/rest-server/config/rest-server.md @@ -15,7 +15,6 @@ There are 2 mandatory config fields in rest-server section: `default-pai-admin-u other config fields are optional, includes: - `server-port: 9186` The port REST server service will listen -- `launcher-type: yarn` The launcher type of REST server, should be "yarn" or "k8s". Default is "yarn" - `jwt-secret: pai-secret` The secret key of JSON web token - `jwt-expire-time` The expire time for a signed jwt token. - `github-owner: Microsoft` The marketplace repo owner in GitHub @@ -31,7 +30,6 @@ After parsing, if you configured the rest-server the model will be like: rest-server: uri: http://rest-server-host:9186/ server-port: 9186 - launcher-type: yarn jwt-secret: pai-secret jwt-expire-time: '7d' default-pai-admin-username: pai-admin @@ -63,12 +61,6 @@ rest-server: cluster_cfg["rest-server"]["server-port"] Int - - rest-server.launcher-type - com["rest-server"]["launcher-type"] - cluster_cfg["rest-server"]["launcher-type"] - String - rest-server.jwt-secret com["rest-server"]["jwt-secret"] diff --git a/src/rest-server/config/rest-server.yaml b/src/rest-server/config/rest-server.yaml index f70c40181f..2a2a15d2ab 100644 --- a/src/rest-server/config/rest-server.yaml +++ b/src/rest-server/config/rest-server.yaml @@ -15,10 +15,9 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +openpai_parser_type: "common" server-port: 9186 -launcher-type: yarn jwt-secret: pai-secret jwt-expire-time: '7d' github-owner: Microsoft diff --git a/src/rest-server/config/rest_server.py b/src/rest-server/config/rest_server.py index 0bd6bb157a..8bfe1dcba7 100644 --- a/src/rest-server/config/rest_server.py +++ b/src/rest-server/config/rest_server.py @@ -29,8 +29,6 @@ def validation_pre(self): return False, '"default-pai-admin-username" is required in rest-server' if 'default-pai-admin-password' not in self.service_configuration: return False, '"default-pai-admin-password" is required in rest-server' - if 'launcher-type' in self.service_configuration and self.service_configuration['launcher-type'] not in ['yarn', 'k8s']: - return False, '"launcher-type" should be "yarn" or "k8s"' try: reservation_time = int(self.service_configuration['debugging-reservation-seconds']) except ValueError: @@ -56,7 +54,6 @@ def run(self): service_object_model['jwt-expire-time'] = self.service_configuration['jwt-expire-time'] service_object_model['default-pai-admin-username'] = self.service_configuration['default-pai-admin-username'] service_object_model['default-pai-admin-password'] = self.service_configuration['default-pai-admin-password'] - service_object_model['launcher-type'] = self.service_configuration['launcher-type'] service_object_model['github-owner'] = self.service_configuration['github-owner'] service_object_model['github-repository'] = self.service_configuration['github-repository'] service_object_model['github-path'] = self.service_configuration['github-path'] diff --git a/src/rest-server/deploy/configmap-create.sh.template b/src/rest-server/deploy/configmap-create.sh.template index 73c279725b..59a1cb5afd 100644 --- a/src/rest-server/deploy/configmap-create.sh.template +++ b/src/rest-server/deploy/configmap-create.sh.template @@ -25,7 +25,7 @@ kubectl create configmap auth-configuration --from-file=auth-configmap/ --dry-r kubectl create configmap group-configuration --from-file=group-configmap/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? -{% if cluster_cfg['rest-server']['launcher-type'] == "yarn" %} +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} kubectl create configmap job-exit-spec-configuration --from-file=job-exit-spec-config/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? {% else %} kubectl create configmap k8s-job-exit-spec-configuration --from-file=k8s-job-exit-spec-config/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? diff --git a/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template b/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template index 16125b5be1..6d527d8da7 100644 --- a/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template +++ b/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {% for spec_item in cluster_cfg['k8s-job-exit-spec']['spec'] %} - code: {{ spec_item['code'] }} {%- if 'phrase' in spec_item %} @@ -59,7 +59,7 @@ {%- if 'controllerPodPatterns' in spec_item['pattern'] %} controllerPodPatterns: {%- for pod_pattern in spec_item['pattern']['controllerPodPatterns'] %} - - + - {%- if 'reasonRegex' in pod_pattern %} reasonRegex: '{{ pod_pattern['reasonRegex'] }}' {%- endif %} @@ -69,7 +69,7 @@ {%- if 'containers' in pod_pattern %} containers: {%- for container in pod_pattern['containers'] %} - - + - {%- if 'nameRegex' in container %} nameRegex: '{{ container['nameRegex'] }}' {%- endif %} @@ -89,7 +89,7 @@ {%- if 'runtimeContainerPatterns' in spec_item['pattern'] %} runtimeContainerPatterns: {%- for runtime_pattern in spec_item['pattern']['runtimeContainerPatterns'] %} - - + - {%- if 'exitCode' in runtime_pattern %} exitCode: {{ runtime_pattern['exitCode'] }} {%- endif %} @@ -103,4 +103,5 @@ {%- endif %} {%- endif %} -{% endfor %} \ No newline at end of file +{% endfor %} +{% endif %} diff --git a/src/rest-server/deploy/rest-server.yaml.template b/src/rest-server/deploy/rest-server.yaml.template index f6d43b6613..110315e0f3 100644 --- a/src/rest-server/deploy/rest-server.yaml.template +++ b/src/rest-server/deploy/rest-server.yaml.template @@ -46,18 +46,18 @@ spec: name: group-configuration-rest-server - mountPath: /job-exit-spec-configuration name: job-exit-spec-rest-server +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {%- if cluster_cfg['hivedscheduler']['config']|length > 1 %} - mountPath: /hived-spec name: hived-spec-rest-server {%- endif %} -{%- if cluster_cfg['rest-server']['launcher-type'] == "k8s" %} - mountPath: /k8s-job-exit-spec-configuration name: k8s-exit-spec-rest-server {%- endif %} env: - name: LAUNCHER_TYPE - value: {{ cluster_cfg['rest-server']['launcher-type'] }} -{%- if cluster_cfg['rest-server']['launcher-type'] == "k8s" %} + value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} - name: LAUNCHER_RUNTIME_IMAGE value: {{ cluster_cfg['cluster']['docker-registry']['prefix'] }}kube-runtime:{{ cluster_cfg['cluster']['docker-registry']['tag'] }} - name: LAUNCHER_RUNTIME_IMAGE_PULL_SECRETS @@ -142,12 +142,12 @@ spec: - name: job-exit-spec-rest-server configMap: name: job-exit-spec-configuration +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {%- if cluster_cfg['hivedscheduler']['config']|length > 1 %} - name: hived-spec-rest-server configMap: name: hivedscheduler-config {%- endif %} -{%- if cluster_cfg['rest-server']['launcher-type'] == "k8s" %} - name: k8s-exit-spec-rest-server configMap: name: k8s-job-exit-spec-configuration diff --git a/src/webportal/config/webportal.k8s.yaml b/src/webportal/config/webportal.k8s.yaml deleted file mode 100644 index dfcb3de20a..0000000000 --- a/src/webportal/config/webportal.k8s.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) Microsoft Corporation -# All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -openpai_parser_type: "k8s" - -server-port: 9286 -log-type: log-manager diff --git a/src/webportal/config/webportal.md b/src/webportal/config/webportal.md index 806f3541d1..44e434b59b 100644 --- a/src/webportal/config/webportal.md +++ b/src/webportal/config/webportal.md @@ -18,7 +18,6 @@ For example, if you want to use different port than the default 9286, add follow ```yaml webportal: server-port: new-value - log-type: yarn ``` About config the web portal plugin, see [PLUGINS.md](../../../docs/webportal/PLUGINS.md) @@ -30,7 +29,6 @@ After parsing, object model looks like: ```yaml webportal: server-port: 9286 - log-type: yarn uri: "http://master_ip:9286" ``` @@ -49,12 +47,6 @@ webportal: cluster_cfg["webportal"]["server-port"] Int - - webportal.log-type - com["webportal"]["log-type"] - cluster_cfg["webportal"]["log-type"] - String - webportal.uri com["webportal"]["uri"] diff --git a/src/webportal/config/webportal.py b/src/webportal/config/webportal.py index 67862d2cf2..1f05ba7291 100644 --- a/src/webportal/config/webportal.py +++ b/src/webportal/config/webportal.py @@ -35,8 +35,6 @@ def validation_pre(self): machine_list = self.cluster_configuration['machine-list'] if len([host for host in machine_list if host.get('pai-master') == 'true']) != 1: return False, '1 and only 1 "pai-master=true" machine is required to deploy the rest server' - if self.service_configuration['log-type'] not in ['yarn', 'log-manager']: - return False, '"log-type" should be yarn or log-manager' return True, None #### Generate the final service object model @@ -58,12 +56,10 @@ def apply_config(plugin): machine_list = self.cluster_configuration['machine-list'] master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0] server_port = self.service_configuration['server-port'] - log_type = self.service_configuration['log-type'] uri = 'http://{0}:{1}'.format(master_ip, server_port) plugins = self.service_configuration['plugins'] return { 'server-port': server_port, - 'log-type': log_type, 'uri': uri, 'plugins': json.dumps([apply_config(plugin) for plugin in plugins]), 'webportal-address': master_ip, diff --git a/src/webportal/config/webportal.yaml b/src/webportal/config/webportal.yaml index 983942d0c3..77cc7423e9 100644 --- a/src/webportal/config/webportal.yaml +++ b/src/webportal/config/webportal.yaml @@ -15,7 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +openpai_parser_type: "common" server-port: 9286 -log-type: yarn diff --git a/src/webportal/deploy/webportal.yaml.template b/src/webportal/deploy/webportal.yaml.template index fb6f4e9fd7..d3328aa2db 100644 --- a/src/webportal/deploy/webportal.yaml.template +++ b/src/webportal/deploy/webportal.yaml.template @@ -37,13 +37,17 @@ spec: imagePullPolicy: Always env: - name: LAUNCHER_TYPE - value: {{ cluster_cfg['rest-server']['launcher-type'] }} + value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }} - name: REST_SERVER_URI value: {{ cluster_cfg['rest-server']['uri'] }} - name: PROMETHEUS_URI value: {{ cluster_cfg['prometheus']['url'] }} - name: LOG_TYPE - value: {{ cluster_cfg['webportal']['log-type'] }} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} + value: log-manager +{%- else %} + value: yarn +{%- endif %} - name: YARN_WEB_PORTAL_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 - name: GRAFANA_URI From dac5cd713a6ceac706aef9f8cb19ce707ba24c2b Mon Sep 17 00:00:00 2001 From: YundongYe Date: Mon, 23 Dec 2019 15:40:06 +0800 Subject: [PATCH 10/17] Update template --- deployment/clusterObjectModel/cluster_object_model.py | 2 +- .../deploy/job-exit-spec-config/job-exit-spec.yaml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index c7d058d10d..f4d00e2f00 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -59,7 +59,7 @@ def get_service_parser(self, service_name, cluster_type): # Prepare Service Configuration layout = self.layout parser_type = "common" - default_service_cfg = [] + default_service_cfg = {} if file_handler.file_exist_or_not(default_path): default_service_cfg = file_handler.load_yaml_config(default_path) if default_service_cfg is not None and "openpai_parser_type" in default_service_cfg: diff --git a/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template b/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template index 35e93b3538..020ddc740b 100644 --- a/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template +++ b/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template @@ -58,4 +58,4 @@ pattern: "{{ spec_item['pattern'] }}" {%- endif %} -{% endfor %} \ No newline at end of file +{% endfor %} From 71beb64fee102b15d313352e9bc1f4daf6a664a2 Mon Sep 17 00:00:00 2001 From: YundongYe <8675883+ydye@users.noreply.github.com> Date: Tue, 24 Dec 2019 14:49:46 +0800 Subject: [PATCH 11/17] [Cluster Objec Model]Don't check hadoop rm uri when in pure k8s cluster. (#4052) * Don't check hadoop rm uri when in pure k8s cluster. --- src/webportal/config/webportal.py | 10 ++++++---- src/webportal/deploy/webportal.yaml.template | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/webportal/config/webportal.py b/src/webportal/config/webportal.py index 1f05ba7291..6b604f067e 100644 --- a/src/webportal/config/webportal.py +++ b/src/webportal/config/webportal.py @@ -67,16 +67,18 @@ def apply_config(plugin): #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness. def validation_post(self, cluster_object_model): - for (service, config) in ( + check_tuple = ( ('rest-server', 'uri'), ('prometheus', 'url'), - ('hadoop-resource-manager', 'master-ip'), ('grafana', 'url'), # TODO - #('kubernetes', 'dashboard-url'), + # ('kubernetes', 'dashboard-url'), ('node-exporter', 'port'), ('prometheus', 'scrape_interval'), - ): + ) + if cluster_object_model['cluster']['common']['cluster-type'] == 'yarn': + check_tuple = (('hadoop-resource-manager', 'master-ip'),)+check_tuple + for (service, config) in check_tuple: if service not in cluster_object_model or config not in cluster_object_model[service]: return False, '{0}.{1} is required'.format(service, config) diff --git a/src/webportal/deploy/webportal.yaml.template b/src/webportal/deploy/webportal.yaml.template index d3328aa2db..5b1224c140 100644 --- a/src/webportal/deploy/webportal.yaml.template +++ b/src/webportal/deploy/webportal.yaml.template @@ -48,8 +48,10 @@ spec: {%- else %} value: yarn {%- endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - name: YARN_WEB_PORTAL_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 +{%- endif %} - name: GRAFANA_URI value: {{ cluster_cfg['grafana']['url'] }} - name: K8S_DASHBOARD_URI From 33c58d91291627f5e3262406d2b965c50adaedd9 Mon Sep 17 00:00:00 2001 From: YundongYe <8675883+ydye@users.noreply.github.com> Date: Wed, 25 Dec 2019 10:01:08 +0800 Subject: [PATCH 12/17] [Rest-server] Add env, volume based on the cluster-type (#4058) --- src/rest-server/config/rest_server.py | 13 +++++++------ .../job-exit-spec.yaml.template | 4 +++- src/rest-server/deploy/rest-server.yaml.template | 6 ++++++ src/rest-server/src/config/yarn.js | 12 +++++++----- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/rest-server/config/rest_server.py b/src/rest-server/config/rest_server.py index 8bfe1dcba7..3e6a41c98b 100644 --- a/src/rest-server/config/rest_server.py +++ b/src/rest-server/config/rest_server.py @@ -65,12 +65,13 @@ def run(self): #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness. def validation_post(self, cluster_object_model): - if 'yarn-frameworklauncher' not in cluster_object_model or 'webservice' not in cluster_object_model['yarn-frameworklauncher']: - return False, 'yarn-frameworklauncher.webservice is required' - if 'hadoop-name-node' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-name-node']: - return False, 'hadoop-name-node.master-ip is required' - if 'hadoop-resource-manager' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-resource-manager']: - return False, 'hadoop-resource-manager.master-ip is required' + if cluster_object_model['cluster']['common']['cluster-type'] == 'yarn': + if 'yarn-frameworklauncher' not in cluster_object_model or 'webservice' not in cluster_object_model['yarn-frameworklauncher']: + return False, 'yarn-frameworklauncher.webservice is required' + if 'hadoop-name-node' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-name-node']: + return False, 'hadoop-name-node.master-ip is required' + if 'hadoop-resource-manager' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-resource-manager']: + return False, 'hadoop-resource-manager.master-ip is required' if 'kubernetes' not in cluster_object_model['layout'] or 'api-servers-url' not in cluster_object_model['layout']['kubernetes']: return False, 'kubernetes.api-servers-url is required' diff --git a/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template b/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template index 020ddc740b..688978ec12 100644 --- a/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template +++ b/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} {% for spec_item in cluster_cfg['job-exit-spec']['spec'] %} - code: {{ spec_item['code'] }} {%- if 'phrase' in spec_item %} @@ -59,3 +59,5 @@ {%- endif %} {% endfor %} +{%- endif %} + diff --git a/src/rest-server/deploy/rest-server.yaml.template b/src/rest-server/deploy/rest-server.yaml.template index 110315e0f3..405155e132 100644 --- a/src/rest-server/deploy/rest-server.yaml.template +++ b/src/rest-server/deploy/rest-server.yaml.template @@ -44,8 +44,10 @@ spec: {% endif %} - mountPath: /group-configuration name: group-configuration-rest-server +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - mountPath: /job-exit-spec-configuration name: job-exit-spec-rest-server +{%- endif %} {%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {%- if cluster_cfg['hivedscheduler']['config']|length > 1 %} - mountPath: /hived-spec @@ -73,6 +75,7 @@ spec: - name: LOG_MANAGER_PORT value: "{{ cluster_cfg['log-manager']['port'] }}" {%- endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - name: LAUNCHER_WEBSERVICE_URI value: {{ cluster_cfg['yarn-frameworklauncher']['webservice'] }} - name: HDFS_URI @@ -81,6 +84,7 @@ spec: value: http://{{ cluster_cfg['hadoop-name-node']['master-ip'] }}:5070 - name: YARN_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 +{%- endif %} - name: JWT_SECRET value: {{ cluster_cfg['rest-server']['jwt-secret'] }} - name: JWT_TOKEN_EXPIRE_TIME @@ -139,9 +143,11 @@ spec: configMap: name: auth-configuration {% endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - name: job-exit-spec-rest-server configMap: name: job-exit-spec-configuration +{%- endif %} {%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {%- if cluster_cfg['hivedscheduler']['config']|length > 1 %} - name: hived-spec-rest-server diff --git a/src/rest-server/src/config/yarn.js b/src/rest-server/src/config/yarn.js index d153217d85..b4598ad04b 100644 --- a/src/rest-server/src/config/yarn.js +++ b/src/rest-server/src/config/yarn.js @@ -18,6 +18,7 @@ // module dependencies const Joi = require('joi'); +const launcherConfig = require('@pai/config/launcher'); // get config from environment variables let yarnConfig = { @@ -53,11 +54,12 @@ const yarnConfigSchema = Joi.object().keys({ .required(), }).required(); -const {error, value} = Joi.validate(yarnConfig, yarnConfigSchema); -if (error) { - throw new Error(`yarn config error\n${error}`); +if (launcherConfig.type === 'yarn') { + const {error, value} = Joi.validate(yarnConfig, yarnConfigSchema); + if (error) { + throw new Error(`yarn config error\n${error}`); + } + yarnConfig = value; } -yarnConfig = value; - module.exports = yarnConfig; From 47fbbd270448d2c14b709ed60b5bdb69448ba035 Mon Sep 17 00:00:00 2001 From: YundongYe <8675883+ydye@users.noreply.github.com> Date: Wed, 25 Dec 2019 14:46:06 +0800 Subject: [PATCH 13/17] Remove yarn config from pylon when cluster-type is k8s (#4063) --- src/pylon/config/pylon.py | 11 ++++++----- .../deploy/pylon-config/location.conf.template | 8 ++++++++ .../deploy/pylon-config/nginx.conf.template | 2 ++ src/pylon/deploy/pylon.yaml.template | 18 ++++++++++++------ 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/pylon/config/pylon.py b/src/pylon/config/pylon.py index 245812e143..592bc9c6bf 100644 --- a/src/pylon/config/pylon.py +++ b/src/pylon/config/pylon.py @@ -56,17 +56,18 @@ def run(self): #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness. def validation_post(self, cluster_object_model): - for (service, config) in ( + check_tuple = ( ('rest-server', 'uri'), - ('hadoop-name-node', 'master-ip'), ('prometheus', 'url'), ('alert-manager', 'url'), # TODO - #('kubernetes', 'dashboard-url'), - ('hadoop-resource-manager', 'master-ip'), + # ('kubernetes', 'dashboard-url'), ('grafana', 'url'), ('webportal', 'uri'), - ): + ) + if cluster_object_model['cluster']['common']['cluster-type'] == 'yarn': + check_tuple = (('hadoop-resource-manager', 'master-ip'), ('hadoop-name-node', 'master-ip'),) + check_tuple + for (service, config) in check_tuple: if service not in cluster_object_model or config not in cluster_object_model[service]: return False, '{0}.{1} is required'.format(service, config) diff --git a/src/pylon/deploy/pylon-config/location.conf.template b/src/pylon/deploy/pylon-config/location.conf.template index 20f91b19e2..9f71b870c6 100644 --- a/src/pylon/deploy/pylon-config/location.conf.template +++ b/src/pylon/deploy/pylon-config/location.conf.template @@ -58,6 +58,7 @@ location ~ ^/kubernetes/api(.*)$ { proxy_pass {{K8S_API_SERVER_URI}}/api$1$is_args$args; } +{% if CLUSTER_TYPE == 'yarn' %} # WebHDFS API server. location ~ ^/webhdfs/api(.*)$ { proxy_pass {{WEBHDFS_URI}}/webhdfs$1$is_args$args; @@ -73,6 +74,7 @@ location @handle_webhdfs_api_redirect { return 300 ""; } } +{% endif %} # Prometheus API server. location ~ ^/prometheus(.*)$ { @@ -89,6 +91,7 @@ location ~ ^/log-manager/([^/]+):(\d+)/(.*)$ { proxy_pass http://$1:$2/log-manager/$3$is_args$args; } +{% if CLUSTER_TYPE == 'yarn' %} # # YARN web portal # @@ -178,6 +181,7 @@ location @handle_yarn_redirect { return 301 $scheme://$http_host/yarn/$1:$2/$3$4; } } +{% endif %} # # Other web portals @@ -192,6 +196,7 @@ location ~ ^/kubernetes-dashboard(.*)$ { proxy_pass {{K8S_DASHBOARD_URI}}$1$is_args$args; } +{% if CLUSTER_TYPE == 'yarn' %} # WebHDFS dashboard location ~ ^/webhdfs$ { # Add '/' to the end of the URL, otherwise there will be a 404 error. @@ -221,6 +226,7 @@ location ~ ^/webhdfs(.*)$ { "get('/startupProgress'" "get('${base}startupProgress'"; } +{% endif %} # Grafana location ~ ^/grafana$ { @@ -268,10 +274,12 @@ location ~ ^(.*)$ { sub_filter '{{PROMETHEUS_URI}}/prometheus' '/prometheus'; +{%- if CLUSTER_TYPE == "yarn" %} sub_filter '{{YARN_WEB_PORTAL_URI}}' '/yarn'; # Add '/' to the end of the URL is used to adapt the envrionent with Azure gateway +{%- endif %} sub_filter '{{K8S_DASHBOARD_URI}}' '/kubernetes-dashboard/'; diff --git a/src/pylon/deploy/pylon-config/nginx.conf.template b/src/pylon/deploy/pylon-config/nginx.conf.template index ae48c2787f..9ea0e4a3d6 100644 --- a/src/pylon/deploy/pylon-config/nginx.conf.template +++ b/src/pylon/deploy/pylon-config/nginx.conf.template @@ -92,6 +92,7 @@ http { } {% endif %} + {% if CLUSTER_TYPE == 'yarn' %} {% if WEBHDFS_LEGACY_PORT %} # Proxy 50070 to webhdfs for backward compatibility server { @@ -122,4 +123,5 @@ http { } } {% endif %} + {% endif %} } diff --git a/src/pylon/deploy/pylon.yaml.template b/src/pylon/deploy/pylon.yaml.template index ee0c936cd2..e62bb94cd5 100644 --- a/src/pylon/deploy/pylon.yaml.template +++ b/src/pylon/deploy/pylon.yaml.template @@ -49,12 +49,8 @@ spec: value: {{ cluster_cfg['rest-server']['uri'] }} - name: K8S_API_SERVER_URI value: {{ cluster_cfg['layout']['kubernetes']['api-servers-url'] }} - - name: WEBHDFS_URI - value: http://{{ cluster_cfg['hadoop-name-node']['master-ip'] }}:5070 -{% if cluster_cfg['pylon']['webhdfs-legacy-port'] %} - - name: WEBHDFS_LEGACY_PORT - value: '{{ cluster_cfg['pylon']['webhdfs-legacy-port'] }}' -{% endif %} + - name: CLUSTER_TYPE + value: cluster_cfg["cluster"]["common"]["cluster-type"] {% if cluster_cfg['authentication']['OIDC'] %} - name: AUTH_MODE value: OIDC @@ -69,8 +65,16 @@ spec: value: {{ cluster_cfg['alert-manager']['url'] }} - name: K8S_DASHBOARD_URI value: {{ cluster_cfg['layout']['kubernetes']['dashboard-url'] }} +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} +{% if cluster_cfg['pylon']['webhdfs-legacy-port'] %} + - name: WEBHDFS_LEGACY_PORT + value: '{{ cluster_cfg['pylon']['webhdfs-legacy-port'] }}' +{% endif %} + - name: WEBHDFS_URI + value: http://{{ cluster_cfg['hadoop-name-node']['master-ip'] }}:5070 - name: YARN_WEB_PORTAL_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 +{% endif %} - name: GRAFANA_URI value: {{ cluster_cfg['grafana']['url'] }} - name: PAI_WEB_PORTAL_URI @@ -82,10 +86,12 @@ spec: - name: pylon-https containerPort: 443 hostPort: 443 +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} {% if cluster_cfg['pylon']['webhdfs-legacy-port'] %} - name: pylon-webhdfs containerPort: {{ cluster_cfg['pylon']['webhdfs-legacy-port'] }} hostPort: {{ cluster_cfg['pylon']['webhdfs-legacy-port'] }} +{% endif %} {% endif %} livenessProbe: httpGet: From 330d6be2156c9109887fcd2bfdf8722213a1938d Mon Sep 17 00:00:00 2001 From: YundongYe Date: Wed, 25 Dec 2019 15:20:40 +0800 Subject: [PATCH 14/17] Remove yarn config from pylon when cluster-type is k8s --- src/rest-server/deploy/stop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rest-server/deploy/stop.sh b/src/rest-server/deploy/stop.sh index 9b679fb426..0e2333a841 100644 --- a/src/rest-server/deploy/stop.sh +++ b/src/rest-server/deploy/stop.sh @@ -27,7 +27,7 @@ if kubectl get configmap | grep -q "auth-configuration"; then kubectl delete configmap auth-configuration || exit $? fi -if kubectl get configmap | grep -q "job-exit-spec-configuration"; then +if kubectl get configmap | grep -q " job-exit-spec-configuration"; then kubectl delete configmap job-exit-spec-configuration || exit $? fi From 3ee9c884b7cb935faddc0227950b6578bc79e38e Mon Sep 17 00:00:00 2001 From: YundongYe <8675883+ydye@users.noreply.github.com> Date: Wed, 25 Dec 2019 16:06:46 +0800 Subject: [PATCH 15/17] [cluster object model] Document update. (#4066) * Cluster Object model update. Add parser type. --- docs/pai-management/doc/how-to-add-new-conm.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/pai-management/doc/how-to-add-new-conm.md b/docs/pai-management/doc/how-to-add-new-conm.md index d19442b5b4..8012d3c1e5 100644 --- a/docs/pai-management/doc/how-to-add-new-conm.md +++ b/docs/pai-management/doc/how-to-add-new-conm.md @@ -35,6 +35,15 @@ src/${service_name}/config/${service_name}.yaml ${service_name}.yaml ```yaml +# The key `openpai_parser_type` is designed for cluster-object-model. And don't use it in other place. +# Now, you could configure this key with one of ['yarn', 'common', 'k8s']. +# When openpai_parser_type is yarn, the service's configuration will be generated if the cluster-type is yarn. +# When openpai_parser_type is k8s, the service's configuration will be generated if the cluster-type is k8s. +# When openpai_parser_type is common, the service's configuration will be generated in both of the cluster-type. +# If the key is missing, default value will be 'common'. +openpai_parser_type: "yarn" + + # key : value service-a-key1: default—value1 From 25682f29ecc670fe17b99cd788a3943e7d4f0cca Mon Sep 17 00:00:00 2001 From: YundongYe Date: Wed, 25 Dec 2019 16:38:11 +0800 Subject: [PATCH 16/17] Cluster Object model update. Add parser type. --- docs/pai-management/doc/how-to-add-new-conm.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/pai-management/doc/how-to-add-new-conm.md b/docs/pai-management/doc/how-to-add-new-conm.md index 8012d3c1e5..c08c279130 100644 --- a/docs/pai-management/doc/how-to-add-new-conm.md +++ b/docs/pai-management/doc/how-to-add-new-conm.md @@ -43,7 +43,6 @@ ${service_name}.yaml # If the key is missing, default value will be 'common'. openpai_parser_type: "yarn" - # key : value service-a-key1: default—value1 From a65cf14b28028e8fd566f8fcc8276d8273b3b3cb Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 26 Dec 2019 15:07:09 +0800 Subject: [PATCH 17/17] From openpai_parser_type to service_type --- .../clusterObjectModel/cluster_object_model.py | 12 ++++++------ docs/pai-management/doc/how-to-add-new-conm.md | 10 +++++----- src/alert-manager/config/alert-manager.yaml | 2 +- src/cleaner/config/cleaner.yaml | 2 +- src/cluster/config/cluster.yaml | 2 +- src/device-plugin/config/device-plugin.yaml | 2 +- src/drivers/config/drivers.yaml | 2 +- src/elasticsearch/config/elasticsearch.yaml | 2 +- src/grafana/config/grafana.yaml | 2 +- src/hadoop-batch-job/config/hadoop-batch-job.yaml | 2 +- src/hadoop-data-node/config/hadoop-data-node.yaml | 2 +- src/hadoop-jobhistory/config/hadoop-jobhistory.yaml | 2 +- src/hadoop-name-node/config/hadoop-name-node.yaml | 2 +- .../config/hadoop-node-manager.yaml | 2 +- .../config/hadoop-resource-manager.yaml | 2 +- src/hivedscheduler/config/hivedscheduler.yaml | 2 +- src/internal-storage/config/internal-storage.yaml | 2 +- src/job-exit-spec/config/job-exit-spec.yaml | 2 +- src/job-exporter/config/job-exporter.yaml | 2 +- src/k8s-dashboard/config/k8s-dashboard.yaml | 2 +- src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml | 2 +- src/log-manager/config/log-manager.yaml | 2 +- src/node-exporter/config/node-exporter.yaml | 2 +- src/postgresql/config/postgresql.yaml | 2 +- src/prometheus/config/prometheus.yaml | 2 +- src/pylon/config/pylon.yaml | 2 +- src/rest-server/config/rest-server.yaml | 2 +- src/storage-manager/config/storage-manager.yaml | 2 +- src/webportal/config/webportal.yaml | 2 +- .../config/yarn-frameworklauncher.yaml | 2 +- src/zookeeper/config/zookeeper.yaml | 2 +- 31 files changed, 40 insertions(+), 40 deletions(-) diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index f4d00e2f00..8075e62770 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -58,20 +58,20 @@ def get_service_parser(self, service_name, cluster_type): # Prepare Service Configuration layout = self.layout - parser_type = "common" + service_type = "common" default_service_cfg = {} if file_handler.file_exist_or_not(default_path): default_service_cfg = file_handler.load_yaml_config(default_path) - if default_service_cfg is not None and "openpai_parser_type" in default_service_cfg: - parser_type = default_service_cfg["openpai_parser_type"] + if default_service_cfg is not None and "service_type" in default_service_cfg: + service_type = default_service_cfg["service_type"] overwrite_service_cfg = {} if self.overwrite_service_configuration is not None and service_name in self.overwrite_service_configuration: overwrite_service_cfg = self.overwrite_service_configuration[service_name] - if "openpai_parser_type" in overwrite_service_cfg: - parser_type = overwrite_service_cfg["openpai_parser_type"] + if "service_type" in overwrite_service_cfg: + service_type = overwrite_service_cfg["service_type"] - if parser_type != "common" and parser_type != cluster_type: + if service_type != "common" and service_type != cluster_type: return None # Init parser instance diff --git a/docs/pai-management/doc/how-to-add-new-conm.md b/docs/pai-management/doc/how-to-add-new-conm.md index c08c279130..af17572303 100644 --- a/docs/pai-management/doc/how-to-add-new-conm.md +++ b/docs/pai-management/doc/how-to-add-new-conm.md @@ -35,13 +35,13 @@ src/${service_name}/config/${service_name}.yaml ${service_name}.yaml ```yaml -# The key `openpai_parser_type` is designed for cluster-object-model. And don't use it in other place. +# The key `service_type` is designed for cluster-object-model. And don't use it in other place. # Now, you could configure this key with one of ['yarn', 'common', 'k8s']. -# When openpai_parser_type is yarn, the service's configuration will be generated if the cluster-type is yarn. -# When openpai_parser_type is k8s, the service's configuration will be generated if the cluster-type is k8s. -# When openpai_parser_type is common, the service's configuration will be generated in both of the cluster-type. +# When service_type is yarn, the service's configuration will be generated if the cluster-type is yarn. +# When service_type is k8s, the service's configuration will be generated if the cluster-type is k8s. +# When service_type is common, the service's configuration will be generated in both of the cluster-type. # If the key is missing, default value will be 'common'. -openpai_parser_type: "yarn" +service_type: "yarn" # key : value service-a-key1: default—value1 diff --git a/src/alert-manager/config/alert-manager.yaml b/src/alert-manager/config/alert-manager.yaml index 2d4ac2eaff..db701b4643 100644 --- a/src/alert-manager/config/alert-manager.yaml +++ b/src/alert-manager/config/alert-manager.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" port: 9093 use-pylon: False diff --git a/src/cleaner/config/cleaner.yaml b/src/cleaner/config/cleaner.yaml index b484b40043..0901eae0ee 100644 --- a/src/cleaner/config/cleaner.yaml +++ b/src/cleaner/config/cleaner.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" threshold: 90 interval: 60 \ No newline at end of file diff --git a/src/cluster/config/cluster.yaml b/src/cluster/config/cluster.yaml index 34fc3b64ae..b79913667a 100644 --- a/src/cluster/config/cluster.yaml +++ b/src/cluster/config/cluster.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" common: cluster-id: pai diff --git a/src/device-plugin/config/device-plugin.yaml b/src/device-plugin/config/device-plugin.yaml index 5654a9bf39..9ce4c9d68a 100644 --- a/src/device-plugin/config/device-plugin.yaml +++ b/src/device-plugin/config/device-plugin.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "k8s" +service_type: "k8s" devices: - "nvidia.com/gpu" diff --git a/src/drivers/config/drivers.yaml b/src/drivers/config/drivers.yaml index 03916af118..59c1d7e79b 100644 --- a/src/drivers/config/drivers.yaml +++ b/src/drivers/config/drivers.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" set-nvidia-runtime: false diff --git a/src/elasticsearch/config/elasticsearch.yaml b/src/elasticsearch/config/elasticsearch.yaml index 2dab086914..3f634f47fa 100644 --- a/src/elasticsearch/config/elasticsearch.yaml +++ b/src/elasticsearch/config/elasticsearch.yaml @@ -15,4 +15,4 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "k8s" +service_type: "k8s" diff --git a/src/grafana/config/grafana.yaml b/src/grafana/config/grafana.yaml index 1509ac1430..6eeea6e18a 100644 --- a/src/grafana/config/grafana.yaml +++ b/src/grafana/config/grafana.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" port: 3000 diff --git a/src/hadoop-batch-job/config/hadoop-batch-job.yaml b/src/hadoop-batch-job/config/hadoop-batch-job.yaml index a3deedd44d..8b4529c591 100644 --- a/src/hadoop-batch-job/config/hadoop-batch-job.yaml +++ b/src/hadoop-batch-job/config/hadoop-batch-job.yaml @@ -15,4 +15,4 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" diff --git a/src/hadoop-data-node/config/hadoop-data-node.yaml b/src/hadoop-data-node/config/hadoop-data-node.yaml index e6a636b397..53afa2900d 100644 --- a/src/hadoop-data-node/config/hadoop-data-node.yaml +++ b/src/hadoop-data-node/config/hadoop-data-node.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" storage_path: diff --git a/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml b/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml index 17babcd520..e2cf5810b9 100644 --- a/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml +++ b/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" diff --git a/src/hadoop-name-node/config/hadoop-name-node.yaml b/src/hadoop-name-node/config/hadoop-name-node.yaml index ac08d35cea..d40c17921c 100644 --- a/src/hadoop-name-node/config/hadoop-name-node.yaml +++ b/src/hadoop-name-node/config/hadoop-name-node.yaml @@ -15,5 +15,5 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" diff --git a/src/hadoop-node-manager/config/hadoop-node-manager.yaml b/src/hadoop-node-manager/config/hadoop-node-manager.yaml index ac08d35cea..d40c17921c 100644 --- a/src/hadoop-node-manager/config/hadoop-node-manager.yaml +++ b/src/hadoop-node-manager/config/hadoop-node-manager.yaml @@ -15,5 +15,5 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml index 0eabd91f49..8401ac0fc0 100644 --- a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml +++ b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" virtualClusters: default: diff --git a/src/hivedscheduler/config/hivedscheduler.yaml b/src/hivedscheduler/config/hivedscheduler.yaml index 2b69070405..293619c1e1 100644 --- a/src/hivedscheduler/config/hivedscheduler.yaml +++ b/src/hivedscheduler/config/hivedscheduler.yaml @@ -16,7 +16,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "k8s" +service_type: "k8s" webservice-port: 30096 diff --git a/src/internal-storage/config/internal-storage.yaml b/src/internal-storage/config/internal-storage.yaml index ef5f39d5d5..f56262e3d1 100644 --- a/src/internal-storage/config/internal-storage.yaml +++ b/src/internal-storage/config/internal-storage.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" enable: false type: hostPath diff --git a/src/job-exit-spec/config/job-exit-spec.yaml b/src/job-exit-spec/config/job-exit-spec.yaml index d1662b2dcd..2bd8e13a16 100644 --- a/src/job-exit-spec/config/job-exit-spec.yaml +++ b/src/job-exit-spec/config/job-exit-spec.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" ################################################################################ # PAI Job ExitSpec Schema diff --git a/src/job-exporter/config/job-exporter.yaml b/src/job-exporter/config/job-exporter.yaml index f1589f2cd0..fbb2a03378 100644 --- a/src/job-exporter/config/job-exporter.yaml +++ b/src/job-exporter/config/job-exporter.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" port: 9102 logging-level: INFO diff --git a/src/k8s-dashboard/config/k8s-dashboard.yaml b/src/k8s-dashboard/config/k8s-dashboard.yaml index 70bd417407..0fb366e5f7 100644 --- a/src/k8s-dashboard/config/k8s-dashboard.yaml +++ b/src/k8s-dashboard/config/k8s-dashboard.yaml @@ -15,4 +15,4 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" \ No newline at end of file +service_type: "common" \ No newline at end of file diff --git a/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml b/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml index 20c8c70901..9cb5166947 100644 --- a/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml +++ b/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "k8s" +service_type: "k8s" ################################################################################ # PAI Job ExitSpec Schema diff --git a/src/log-manager/config/log-manager.yaml b/src/log-manager/config/log-manager.yaml index cc015283d9..1fd2af76a5 100644 --- a/src/log-manager/config/log-manager.yaml +++ b/src/log-manager/config/log-manager.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "k8s" +service_type: "k8s" port: 9103 diff --git a/src/node-exporter/config/node-exporter.yaml b/src/node-exporter/config/node-exporter.yaml index bb3fd558c9..5828131305 100644 --- a/src/node-exporter/config/node-exporter.yaml +++ b/src/node-exporter/config/node-exporter.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" port: 9100 diff --git a/src/postgresql/config/postgresql.yaml b/src/postgresql/config/postgresql.yaml index 8e136f082b..d194624f4c 100644 --- a/src/postgresql/config/postgresql.yaml +++ b/src/postgresql/config/postgresql.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" enable: false user: root diff --git a/src/prometheus/config/prometheus.yaml b/src/prometheus/config/prometheus.yaml index cd8acce469..e9291f8325 100644 --- a/src/prometheus/config/prometheus.yaml +++ b/src/prometheus/config/prometheus.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" port: 9091 scrape_interval: 30 diff --git a/src/pylon/config/pylon.yaml b/src/pylon/config/pylon.yaml index a4434e2753..3a33263d54 100644 --- a/src/pylon/config/pylon.yaml +++ b/src/pylon/config/pylon.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" port: 80 # The following 2 config is for internal use diff --git a/src/rest-server/config/rest-server.yaml b/src/rest-server/config/rest-server.yaml index 2a2a15d2ab..12f5ac1efe 100644 --- a/src/rest-server/config/rest-server.yaml +++ b/src/rest-server/config/rest-server.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" server-port: 9186 jwt-secret: pai-secret diff --git a/src/storage-manager/config/storage-manager.yaml b/src/storage-manager/config/storage-manager.yaml index 8e05dfd8f1..b58de72f70 100644 --- a/src/storage-manager/config/storage-manager.yaml +++ b/src/storage-manager/config/storage-manager.yaml @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" # Global config localpath: /share diff --git a/src/webportal/config/webportal.yaml b/src/webportal/config/webportal.yaml index 77cc7423e9..490002d545 100644 --- a/src/webportal/config/webportal.yaml +++ b/src/webportal/config/webportal.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "common" +service_type: "common" server-port: 9286 diff --git a/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml b/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml index 8607bb2175..a25baac66a 100644 --- a/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml +++ b/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn" frameworklauncher-port: 9086 \ No newline at end of file diff --git a/src/zookeeper/config/zookeeper.yaml b/src/zookeeper/config/zookeeper.yaml index 17babcd520..e2cf5810b9 100644 --- a/src/zookeeper/config/zookeeper.yaml +++ b/src/zookeeper/config/zookeeper.yaml @@ -15,6 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -openpai_parser_type: "yarn" +service_type: "yarn"