diff --git a/deployment/clusterObjectModel/cluster_object_model.py b/deployment/clusterObjectModel/cluster_object_model.py index f4c092316d..8075e62770 100644 --- a/deployment/clusterObjectModel/cluster_object_model.py +++ b/deployment/clusterObjectModel/cluster_object_model.py @@ -54,20 +54,25 @@ def get_service_model_list(self): def get_service_parser(self, service_name, cluster_type): sys.path.insert(0, '{0}/../../src/{1}/config'.format(package_directory_com, service_name)) - default_path = "{0}/../../src/{1}/config/{1}.{2}.yaml".format(package_directory_com, service_name, cluster_type) - if not file_handler.file_exist_or_not(default_path): - default_path = "{0}/../../src/{1}/config/{1}.yaml".format(package_directory_com, service_name) + default_path = "{0}/../../src/{1}/config/{1}.yaml".format(package_directory_com, service_name) # Prepare Service Configuration layout = self.layout - - default_service_cfg = [] + service_type = "common" + default_service_cfg = {} if file_handler.file_exist_or_not(default_path): default_service_cfg = file_handler.load_yaml_config(default_path) + if default_service_cfg is not None and "service_type" in default_service_cfg: + service_type = default_service_cfg["service_type"] overwrite_service_cfg = {} if self.overwrite_service_configuration is not None and service_name in self.overwrite_service_configuration: overwrite_service_cfg = self.overwrite_service_configuration[service_name] + if "service_type" in overwrite_service_cfg: + service_type = overwrite_service_cfg["service_type"] + + if service_type != "common" and service_type != cluster_type: + return None # Init parser instance parser_module = importlib.import_module(service_name.replace("-", "_")) @@ -128,7 +133,11 @@ def service_config(self): service_model_list = self.get_service_model_list() for service_name in service_model_list: - parser_dict[service_name] = self.get_service_parser(service_name, cluster_type) + parser = self.get_service_parser(service_name, cluster_type) + if parser is None: + continue + parser_dict[service_name] = parser + return self.load_config(parser_dict) def kubernetes_config(self): diff --git a/deployment/clusterObjectModel/test/test_template_generate.py b/deployment/clusterObjectModel/test/test_template_generate.py index 130c62fc27..a4cef5e817 100644 --- a/deployment/clusterObjectModel/test/test_template_generate.py +++ b/deployment/clusterObjectModel/test/test_template_generate.py @@ -67,11 +67,12 @@ def test_template_generate(self): subdir_list = directory_handler.get_subdirectory_list(src_path) for subdir in subdir_list: - service_deploy_dir = "{0}/{1}/deploy".format(src_path, subdir) - service_deploy_conf_path = "{0}/{1}/deploy/service.yaml".format(src_path, subdir) + service_deploy_conf_path = "{0}/{1}/deploy/service.yaml".format(src_path, subdir) if file_handler.directory_exits(service_deploy_dir) and file_handler.file_exist_or_not(service_deploy_conf_path): - service_list.append(subdir) + service_conf = file_handler.load_yaml_config(service_deploy_conf_path) + if ("cluster-type" not in service_conf) or ("cluster-type" in service_conf and "yarn" in service_conf["cluster-type"]): + service_list.append(subdir) for serv in service_list: service_conf = file_handler.load_yaml_config("{0}/{1}/deploy/service.yaml".format(src_path, serv)) diff --git a/deployment/quick-start/services-configuration.yaml.template b/deployment/quick-start/services-configuration.yaml.template index 562f33f281..2af5fc86b4 100644 --- a/deployment/quick-start/services-configuration.yaml.template +++ b/deployment/quick-start/services-configuration.yaml.template @@ -65,8 +65,6 @@ cluster: rest-server: -# # launcher type. k8s or yarn -# launcher-type: k8s # database admin username default-pai-admin-username: admin # database admin password @@ -76,12 +74,13 @@ rest-server: # uncomment following section if you want to customize the port of web portal # webportal: # server-port: 9286 -# log-type: yarn + # uncomment following if you want to change customeize grafana # grafana: # port: 3000 + # uncomment following if you want to change customeize drivers #drivers: # set-nvidia-runtime: false diff --git a/docs/pai-management/doc/how-to-add-new-conm.md b/docs/pai-management/doc/how-to-add-new-conm.md index d19442b5b4..af17572303 100644 --- a/docs/pai-management/doc/how-to-add-new-conm.md +++ b/docs/pai-management/doc/how-to-add-new-conm.md @@ -35,6 +35,14 @@ src/${service_name}/config/${service_name}.yaml ${service_name}.yaml ```yaml +# The key `service_type` is designed for cluster-object-model. And don't use it in other place. +# Now, you could configure this key with one of ['yarn', 'common', 'k8s']. +# When service_type is yarn, the service's configuration will be generated if the cluster-type is yarn. +# When service_type is k8s, the service's configuration will be generated if the cluster-type is k8s. +# When service_type is common, the service's configuration will be generated in both of the cluster-type. +# If the key is missing, default value will be 'common'. +service_type: "yarn" + # key : value service-a-key1: default—value1 diff --git a/src/alert-manager/config/alert-manager.yaml b/src/alert-manager/config/alert-manager.yaml index bb4b426b6a..db701b4643 100644 --- a/src/alert-manager/config/alert-manager.yaml +++ b/src/alert-manager/config/alert-manager.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" + port: 9093 use-pylon: False -repeat-interval: '24h' +repeat-interval: '24h' \ No newline at end of file diff --git a/src/cleaner/config/cleaner.yaml b/src/cleaner/config/cleaner.yaml index bd053d9cfe..0901eae0ee 100644 --- a/src/cleaner/config/cleaner.yaml +++ b/src/cleaner/config/cleaner.yaml @@ -15,5 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + threshold: 90 interval: 60 \ No newline at end of file diff --git a/src/cluster/config/cluster.yaml b/src/cluster/config/cluster.yaml index 29e4cf94a3..b79913667a 100644 --- a/src/cluster/config/cluster.yaml +++ b/src/cluster/config/cluster.yaml @@ -15,6 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "common" common: cluster-id: pai diff --git a/src/device-plugin/config/device-plugin.yaml b/src/device-plugin/config/device-plugin.yaml index 7c89e66054..9ce4c9d68a 100644 --- a/src/device-plugin/config/device-plugin.yaml +++ b/src/device-plugin/config/device-plugin.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "k8s" + devices: - "nvidia.com/gpu" - "rdma/hca" diff --git a/src/drivers/config/drivers.yaml b/src/drivers/config/drivers.yaml index c538f1d8c5..59c1d7e79b 100644 --- a/src/drivers/config/drivers.yaml +++ b/src/drivers/config/drivers.yaml @@ -15,6 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" set-nvidia-runtime: false diff --git a/src/webportal/config/webportal.k8s.yaml b/src/elasticsearch/config/elasticsearch.yaml similarity index 96% rename from src/webportal/config/webportal.k8s.yaml rename to src/elasticsearch/config/elasticsearch.yaml index b2be8e1997..3f634f47fa 100644 --- a/src/webportal/config/webportal.k8s.yaml +++ b/src/elasticsearch/config/elasticsearch.yaml @@ -15,5 +15,4 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -server-port: 9286 -log-type: log-manager \ No newline at end of file +service_type: "k8s" diff --git a/src/grafana/config/grafana.yaml b/src/grafana/config/grafana.yaml index 87c3f5a1c1..6eeea6e18a 100644 --- a/src/grafana/config/grafana.yaml +++ b/src/grafana/config/grafana.yaml @@ -1 +1,20 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" + port: 3000 diff --git a/src/hadoop-batch-job/config/hadoop-batch-job.yaml b/src/hadoop-batch-job/config/hadoop-batch-job.yaml index dbde8d9a14..8b4529c591 100644 --- a/src/hadoop-batch-job/config/hadoop-batch-job.yaml +++ b/src/hadoop-batch-job/config/hadoop-batch-job.yaml @@ -15,3 +15,4 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" diff --git a/src/hadoop-data-node/config/hadoop-data-node.yaml b/src/hadoop-data-node/config/hadoop-data-node.yaml index 0be1c28332..53afa2900d 100644 --- a/src/hadoop-data-node/config/hadoop-data-node.yaml +++ b/src/hadoop-data-node/config/hadoop-data-node.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + storage_path: diff --git a/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml b/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml index dbde8d9a14..e2cf5810b9 100644 --- a/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml +++ b/src/hadoop-jobhistory/config/hadoop-jobhistory.yaml @@ -15,3 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + + diff --git a/src/hadoop-name-node/config/hadoop-name-node.yaml b/src/hadoop-name-node/config/hadoop-name-node.yaml index dbde8d9a14..d40c17921c 100644 --- a/src/hadoop-name-node/config/hadoop-name-node.yaml +++ b/src/hadoop-name-node/config/hadoop-name-node.yaml @@ -15,3 +15,5 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + diff --git a/src/hadoop-node-manager/config/hadoop-node-manager.yaml b/src/hadoop-node-manager/config/hadoop-node-manager.yaml index dbde8d9a14..d40c17921c 100644 --- a/src/hadoop-node-manager/config/hadoop-node-manager.yaml +++ b/src/hadoop-node-manager/config/hadoop-node-manager.yaml @@ -15,3 +15,5 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + diff --git a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml index 2330e758e1..8401ac0fc0 100644 --- a/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml +++ b/src/hadoop-resource-manager/config/hadoop-resource-manager.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + virtualClusters: default: description: Default VC. diff --git a/src/hivedscheduler/config/hivedscheduler.yaml b/src/hivedscheduler/config/hivedscheduler.yaml index 1047c11cbb..293619c1e1 100644 --- a/src/hivedscheduler/config/hivedscheduler.yaml +++ b/src/hivedscheduler/config/hivedscheduler.yaml @@ -15,4 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "k8s" + webservice-port: 30096 + diff --git a/src/internal-storage/config/internal-storage.yaml b/src/internal-storage/config/internal-storage.yaml index 4c6baf2d70..f56262e3d1 100644 --- a/src/internal-storage/config/internal-storage.yaml +++ b/src/internal-storage/config/internal-storage.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" + enable: false type: hostPath root-path: /mnt/paiInternal diff --git a/src/job-exit-spec/config/job-exit-spec.yaml b/src/job-exit-spec/config/job-exit-spec.yaml index e9d6dbaad6..2bd8e13a16 100644 --- a/src/job-exit-spec/config/job-exit-spec.yaml +++ b/src/job-exit-spec/config/job-exit-spec.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + ################################################################################ # PAI Job ExitSpec Schema ################################################################################ diff --git a/src/job-exporter/config/job-exporter.yaml b/src/job-exporter/config/job-exporter.yaml index e895fafc00..fbb2a03378 100644 --- a/src/job-exporter/config/job-exporter.yaml +++ b/src/job-exporter/config/job-exporter.yaml @@ -1,19 +1,21 @@ # Copyright (c) Microsoft Corporation -# # All rights reserved. -# # -# # MIT License -# # -# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# # documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# # to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# # -# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" port: 9102 logging-level: INFO diff --git a/src/job-exporter/deploy/job-exporter.yaml.template b/src/job-exporter/deploy/job-exporter.yaml.template index a8a16cc79d..a685c62f59 100644 --- a/src/job-exporter/deploy/job-exporter.yaml.template +++ b/src/job-exporter/deploy/job-exporter.yaml.template @@ -71,7 +71,7 @@ spec: - name: NVIDIA_VISIBLE_DEVICES value: all - name: LAUNCHER_TYPE - value: {{ cluster_cfg['rest-server']['launcher-type'] }} + value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }} {%- if cluster_cfg['cluster']['common']['deploy-in-aks'] == "true" %} - name: DEPLOY_ENV value: aks diff --git a/src/k8s-dashboard/config/k8s-dashboard.yaml b/src/k8s-dashboard/config/k8s-dashboard.yaml index afedca73fa..0fb366e5f7 100644 --- a/src/k8s-dashboard/config/k8s-dashboard.yaml +++ b/src/k8s-dashboard/config/k8s-dashboard.yaml @@ -14,3 +14,5 @@ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" \ No newline at end of file diff --git a/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml b/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml index 7d3b7b095d..9cb5166947 100644 --- a/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml +++ b/src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "k8s" + ################################################################################ # PAI Job ExitSpec Schema ################################################################################ diff --git a/src/log-manager/config/log-manager.yaml b/src/log-manager/config/log-manager.yaml index ccfa273b9e..1fd2af76a5 100644 --- a/src/log-manager/config/log-manager.yaml +++ b/src/log-manager/config/log-manager.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "k8s" + port: 9103 diff --git a/src/node-exporter/config/node-exporter.yaml b/src/node-exporter/config/node-exporter.yaml index e5f0bf30b7..5828131305 100644 --- a/src/node-exporter/config/node-exporter.yaml +++ b/src/node-exporter/config/node-exporter.yaml @@ -1,18 +1,20 @@ # Copyright (c) Microsoft Corporation -# # All rights reserved. -# # -# # MIT License -# # -# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# # documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# # to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# # -# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" port: 9100 diff --git a/src/postgresql/config/postgresql.yaml b/src/postgresql/config/postgresql.yaml index edcb1a503e..d194624f4c 100644 --- a/src/postgresql/config/postgresql.yaml +++ b/src/postgresql/config/postgresql.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" + enable: false user: root passwd: rootpass diff --git a/src/prometheus/config/prometheus.yaml b/src/prometheus/config/prometheus.yaml index 24b45f0a50..e9291f8325 100644 --- a/src/prometheus/config/prometheus.yaml +++ b/src/prometheus/config/prometheus.yaml @@ -1,2 +1,21 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" + port: 9091 scrape_interval: 30 diff --git a/src/pylon/config/pylon.py b/src/pylon/config/pylon.py index 245812e143..592bc9c6bf 100644 --- a/src/pylon/config/pylon.py +++ b/src/pylon/config/pylon.py @@ -56,17 +56,18 @@ def run(self): #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness. def validation_post(self, cluster_object_model): - for (service, config) in ( + check_tuple = ( ('rest-server', 'uri'), - ('hadoop-name-node', 'master-ip'), ('prometheus', 'url'), ('alert-manager', 'url'), # TODO - #('kubernetes', 'dashboard-url'), - ('hadoop-resource-manager', 'master-ip'), + # ('kubernetes', 'dashboard-url'), ('grafana', 'url'), ('webportal', 'uri'), - ): + ) + if cluster_object_model['cluster']['common']['cluster-type'] == 'yarn': + check_tuple = (('hadoop-resource-manager', 'master-ip'), ('hadoop-name-node', 'master-ip'),) + check_tuple + for (service, config) in check_tuple: if service not in cluster_object_model or config not in cluster_object_model[service]: return False, '{0}.{1} is required'.format(service, config) diff --git a/src/pylon/config/pylon.yaml b/src/pylon/config/pylon.yaml index 0278928e96..3a33263d54 100644 --- a/src/pylon/config/pylon.yaml +++ b/src/pylon/config/pylon.yaml @@ -1,3 +1,22 @@ +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +service_type: "common" + port: 80 # The following 2 config is for internal use webhdfs-legacy-port: 50070 diff --git a/src/pylon/deploy/pylon-config/location.conf.template b/src/pylon/deploy/pylon-config/location.conf.template index 20f91b19e2..9f71b870c6 100644 --- a/src/pylon/deploy/pylon-config/location.conf.template +++ b/src/pylon/deploy/pylon-config/location.conf.template @@ -58,6 +58,7 @@ location ~ ^/kubernetes/api(.*)$ { proxy_pass {{K8S_API_SERVER_URI}}/api$1$is_args$args; } +{% if CLUSTER_TYPE == 'yarn' %} # WebHDFS API server. location ~ ^/webhdfs/api(.*)$ { proxy_pass {{WEBHDFS_URI}}/webhdfs$1$is_args$args; @@ -73,6 +74,7 @@ location @handle_webhdfs_api_redirect { return 300 ""; } } +{% endif %} # Prometheus API server. location ~ ^/prometheus(.*)$ { @@ -89,6 +91,7 @@ location ~ ^/log-manager/([^/]+):(\d+)/(.*)$ { proxy_pass http://$1:$2/log-manager/$3$is_args$args; } +{% if CLUSTER_TYPE == 'yarn' %} # # YARN web portal # @@ -178,6 +181,7 @@ location @handle_yarn_redirect { return 301 $scheme://$http_host/yarn/$1:$2/$3$4; } } +{% endif %} # # Other web portals @@ -192,6 +196,7 @@ location ~ ^/kubernetes-dashboard(.*)$ { proxy_pass {{K8S_DASHBOARD_URI}}$1$is_args$args; } +{% if CLUSTER_TYPE == 'yarn' %} # WebHDFS dashboard location ~ ^/webhdfs$ { # Add '/' to the end of the URL, otherwise there will be a 404 error. @@ -221,6 +226,7 @@ location ~ ^/webhdfs(.*)$ { "get('/startupProgress'" "get('${base}startupProgress'"; } +{% endif %} # Grafana location ~ ^/grafana$ { @@ -268,10 +274,12 @@ location ~ ^(.*)$ { sub_filter '{{PROMETHEUS_URI}}/prometheus' '/prometheus'; +{%- if CLUSTER_TYPE == "yarn" %} sub_filter '{{YARN_WEB_PORTAL_URI}}' '/yarn'; # Add '/' to the end of the URL is used to adapt the envrionent with Azure gateway +{%- endif %} sub_filter '{{K8S_DASHBOARD_URI}}' '/kubernetes-dashboard/'; diff --git a/src/pylon/deploy/pylon-config/nginx.conf.template b/src/pylon/deploy/pylon-config/nginx.conf.template index ae48c2787f..9ea0e4a3d6 100644 --- a/src/pylon/deploy/pylon-config/nginx.conf.template +++ b/src/pylon/deploy/pylon-config/nginx.conf.template @@ -92,6 +92,7 @@ http { } {% endif %} + {% if CLUSTER_TYPE == 'yarn' %} {% if WEBHDFS_LEGACY_PORT %} # Proxy 50070 to webhdfs for backward compatibility server { @@ -122,4 +123,5 @@ http { } } {% endif %} + {% endif %} } diff --git a/src/pylon/deploy/pylon.yaml.template b/src/pylon/deploy/pylon.yaml.template index ee0c936cd2..e62bb94cd5 100644 --- a/src/pylon/deploy/pylon.yaml.template +++ b/src/pylon/deploy/pylon.yaml.template @@ -49,12 +49,8 @@ spec: value: {{ cluster_cfg['rest-server']['uri'] }} - name: K8S_API_SERVER_URI value: {{ cluster_cfg['layout']['kubernetes']['api-servers-url'] }} - - name: WEBHDFS_URI - value: http://{{ cluster_cfg['hadoop-name-node']['master-ip'] }}:5070 -{% if cluster_cfg['pylon']['webhdfs-legacy-port'] %} - - name: WEBHDFS_LEGACY_PORT - value: '{{ cluster_cfg['pylon']['webhdfs-legacy-port'] }}' -{% endif %} + - name: CLUSTER_TYPE + value: cluster_cfg["cluster"]["common"]["cluster-type"] {% if cluster_cfg['authentication']['OIDC'] %} - name: AUTH_MODE value: OIDC @@ -69,8 +65,16 @@ spec: value: {{ cluster_cfg['alert-manager']['url'] }} - name: K8S_DASHBOARD_URI value: {{ cluster_cfg['layout']['kubernetes']['dashboard-url'] }} +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} +{% if cluster_cfg['pylon']['webhdfs-legacy-port'] %} + - name: WEBHDFS_LEGACY_PORT + value: '{{ cluster_cfg['pylon']['webhdfs-legacy-port'] }}' +{% endif %} + - name: WEBHDFS_URI + value: http://{{ cluster_cfg['hadoop-name-node']['master-ip'] }}:5070 - name: YARN_WEB_PORTAL_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 +{% endif %} - name: GRAFANA_URI value: {{ cluster_cfg['grafana']['url'] }} - name: PAI_WEB_PORTAL_URI @@ -82,10 +86,12 @@ spec: - name: pylon-https containerPort: 443 hostPort: 443 +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} {% if cluster_cfg['pylon']['webhdfs-legacy-port'] %} - name: pylon-webhdfs containerPort: {{ cluster_cfg['pylon']['webhdfs-legacy-port'] }} hostPort: {{ cluster_cfg['pylon']['webhdfs-legacy-port'] }} +{% endif %} {% endif %} livenessProbe: httpGet: diff --git a/src/rest-server/config/rest-server.k8s.yaml b/src/rest-server/config/rest-server.k8s.yaml deleted file mode 100644 index c91461e3b1..0000000000 --- a/src/rest-server/config/rest-server.k8s.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) Microsoft Corporation -# All rights reserved. -# -# MIT License -# -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -server-port: 9186 -launcher-type: k8s -jwt-secret: pai-secret -jwt-expire-time: '7d' -github-owner: Microsoft -github-repository: pai -github-path: marketplace -debugging-reservation-seconds: 604800 diff --git a/src/rest-server/config/rest-server.md b/src/rest-server/config/rest-server.md index 48cd8fce4d..79d5892272 100644 --- a/src/rest-server/config/rest-server.md +++ b/src/rest-server/config/rest-server.md @@ -15,7 +15,6 @@ There are 2 mandatory config fields in rest-server section: `default-pai-admin-u other config fields are optional, includes: - `server-port: 9186` The port REST server service will listen -- `launcher-type: yarn` The launcher type of REST server, should be "yarn" or "k8s". Default is "yarn" - `jwt-secret: pai-secret` The secret key of JSON web token - `jwt-expire-time` The expire time for a signed jwt token. - `github-owner: Microsoft` The marketplace repo owner in GitHub @@ -31,7 +30,6 @@ After parsing, if you configured the rest-server the model will be like: rest-server: uri: http://rest-server-host:9186/ server-port: 9186 - launcher-type: yarn jwt-secret: pai-secret jwt-expire-time: '7d' default-pai-admin-username: pai-admin @@ -63,12 +61,6 @@ rest-server: cluster_cfg["rest-server"]["server-port"] Int - - rest-server.launcher-type - com["rest-server"]["launcher-type"] - cluster_cfg["rest-server"]["launcher-type"] - String - rest-server.jwt-secret com["rest-server"]["jwt-secret"] diff --git a/src/rest-server/config/rest-server.yaml b/src/rest-server/config/rest-server.yaml index 2cd616b233..12f5ac1efe 100644 --- a/src/rest-server/config/rest-server.yaml +++ b/src/rest-server/config/rest-server.yaml @@ -15,8 +15,9 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "common" + server-port: 9186 -launcher-type: yarn jwt-secret: pai-secret jwt-expire-time: '7d' github-owner: Microsoft diff --git a/src/rest-server/config/rest_server.py b/src/rest-server/config/rest_server.py index 0bd6bb157a..3e6a41c98b 100644 --- a/src/rest-server/config/rest_server.py +++ b/src/rest-server/config/rest_server.py @@ -29,8 +29,6 @@ def validation_pre(self): return False, '"default-pai-admin-username" is required in rest-server' if 'default-pai-admin-password' not in self.service_configuration: return False, '"default-pai-admin-password" is required in rest-server' - if 'launcher-type' in self.service_configuration and self.service_configuration['launcher-type'] not in ['yarn', 'k8s']: - return False, '"launcher-type" should be "yarn" or "k8s"' try: reservation_time = int(self.service_configuration['debugging-reservation-seconds']) except ValueError: @@ -56,7 +54,6 @@ def run(self): service_object_model['jwt-expire-time'] = self.service_configuration['jwt-expire-time'] service_object_model['default-pai-admin-username'] = self.service_configuration['default-pai-admin-username'] service_object_model['default-pai-admin-password'] = self.service_configuration['default-pai-admin-password'] - service_object_model['launcher-type'] = self.service_configuration['launcher-type'] service_object_model['github-owner'] = self.service_configuration['github-owner'] service_object_model['github-repository'] = self.service_configuration['github-repository'] service_object_model['github-path'] = self.service_configuration['github-path'] @@ -68,12 +65,13 @@ def run(self): #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness. def validation_post(self, cluster_object_model): - if 'yarn-frameworklauncher' not in cluster_object_model or 'webservice' not in cluster_object_model['yarn-frameworklauncher']: - return False, 'yarn-frameworklauncher.webservice is required' - if 'hadoop-name-node' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-name-node']: - return False, 'hadoop-name-node.master-ip is required' - if 'hadoop-resource-manager' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-resource-manager']: - return False, 'hadoop-resource-manager.master-ip is required' + if cluster_object_model['cluster']['common']['cluster-type'] == 'yarn': + if 'yarn-frameworklauncher' not in cluster_object_model or 'webservice' not in cluster_object_model['yarn-frameworklauncher']: + return False, 'yarn-frameworklauncher.webservice is required' + if 'hadoop-name-node' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-name-node']: + return False, 'hadoop-name-node.master-ip is required' + if 'hadoop-resource-manager' not in cluster_object_model or 'master-ip' not in cluster_object_model['hadoop-resource-manager']: + return False, 'hadoop-resource-manager.master-ip is required' if 'kubernetes' not in cluster_object_model['layout'] or 'api-servers-url' not in cluster_object_model['layout']['kubernetes']: return False, 'kubernetes.api-servers-url is required' diff --git a/src/rest-server/deploy/configmap-create.sh.template b/src/rest-server/deploy/configmap-create.sh.template index a494b84108..59a1cb5afd 100644 --- a/src/rest-server/deploy/configmap-create.sh.template +++ b/src/rest-server/deploy/configmap-create.sh.template @@ -25,8 +25,10 @@ kubectl create configmap auth-configuration --from-file=auth-configmap/ --dry-r kubectl create configmap group-configuration --from-file=group-configmap/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} kubectl create configmap job-exit-spec-configuration --from-file=job-exit-spec-config/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? - +{% else %} kubectl create configmap k8s-job-exit-spec-configuration --from-file=k8s-job-exit-spec-config/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $? +{% endif %} popd > /dev/null diff --git a/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template b/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template index 35e93b3538..688978ec12 100644 --- a/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template +++ b/src/rest-server/deploy/job-exit-spec-config/job-exit-spec.yaml.template @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} {% for spec_item in cluster_cfg['job-exit-spec']['spec'] %} - code: {{ spec_item['code'] }} {%- if 'phrase' in spec_item %} @@ -58,4 +58,6 @@ pattern: "{{ spec_item['pattern'] }}" {%- endif %} -{% endfor %} \ No newline at end of file +{% endfor %} +{%- endif %} + diff --git a/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template b/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template index 16125b5be1..6d527d8da7 100644 --- a/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template +++ b/src/rest-server/deploy/k8s-job-exit-spec-config/k8s-job-exit-spec.yaml.template @@ -15,7 +15,7 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - +{% if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {% for spec_item in cluster_cfg['k8s-job-exit-spec']['spec'] %} - code: {{ spec_item['code'] }} {%- if 'phrase' in spec_item %} @@ -59,7 +59,7 @@ {%- if 'controllerPodPatterns' in spec_item['pattern'] %} controllerPodPatterns: {%- for pod_pattern in spec_item['pattern']['controllerPodPatterns'] %} - - + - {%- if 'reasonRegex' in pod_pattern %} reasonRegex: '{{ pod_pattern['reasonRegex'] }}' {%- endif %} @@ -69,7 +69,7 @@ {%- if 'containers' in pod_pattern %} containers: {%- for container in pod_pattern['containers'] %} - - + - {%- if 'nameRegex' in container %} nameRegex: '{{ container['nameRegex'] }}' {%- endif %} @@ -89,7 +89,7 @@ {%- if 'runtimeContainerPatterns' in spec_item['pattern'] %} runtimeContainerPatterns: {%- for runtime_pattern in spec_item['pattern']['runtimeContainerPatterns'] %} - - + - {%- if 'exitCode' in runtime_pattern %} exitCode: {{ runtime_pattern['exitCode'] }} {%- endif %} @@ -103,4 +103,5 @@ {%- endif %} {%- endif %} -{% endfor %} \ No newline at end of file +{% endfor %} +{% endif %} diff --git a/src/rest-server/deploy/rest-server.yaml.template b/src/rest-server/deploy/rest-server.yaml.template index fd8abad0a7..405155e132 100644 --- a/src/rest-server/deploy/rest-server.yaml.template +++ b/src/rest-server/deploy/rest-server.yaml.template @@ -44,20 +44,22 @@ spec: {% endif %} - mountPath: /group-configuration name: group-configuration-rest-server +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - mountPath: /job-exit-spec-configuration name: job-exit-spec-rest-server +{%- endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {%- if cluster_cfg['hivedscheduler']['config']|length > 1 %} - mountPath: /hived-spec name: hived-spec-rest-server {%- endif %} -{%- if cluster_cfg['rest-server']['launcher-type'] == "k8s" %} - mountPath: /k8s-job-exit-spec-configuration name: k8s-exit-spec-rest-server {%- endif %} env: - name: LAUNCHER_TYPE - value: {{ cluster_cfg['rest-server']['launcher-type'] }} -{%- if cluster_cfg['rest-server']['launcher-type'] == "k8s" %} + value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} - name: LAUNCHER_RUNTIME_IMAGE value: {{ cluster_cfg['cluster']['docker-registry']['prefix'] }}kube-runtime:{{ cluster_cfg['cluster']['docker-registry']['tag'] }} - name: LAUNCHER_RUNTIME_IMAGE_PULL_SECRETS @@ -73,6 +75,7 @@ spec: - name: LOG_MANAGER_PORT value: "{{ cluster_cfg['log-manager']['port'] }}" {%- endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - name: LAUNCHER_WEBSERVICE_URI value: {{ cluster_cfg['yarn-frameworklauncher']['webservice'] }} - name: HDFS_URI @@ -81,6 +84,7 @@ spec: value: http://{{ cluster_cfg['hadoop-name-node']['master-ip'] }}:5070 - name: YARN_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 +{%- endif %} - name: JWT_SECRET value: {{ cluster_cfg['rest-server']['jwt-secret'] }} - name: JWT_TOKEN_EXPIRE_TIME @@ -98,7 +102,7 @@ spec: - name: AUTHN_METHOD value: OIDC {% endif %} -{%- if cluster_cfg['elasticsearch']['master-ip'] %} +{%- if 'elasticsearch' in cluster_cfg and cluster_cfg['elasticsearch']['master-ip'] %} - name: ELASTICSEARCH_URI value: http://{{ cluster_cfg['elasticsearch']['master-ip'] }}:30100 {%- endif %} @@ -139,15 +143,17 @@ spec: configMap: name: auth-configuration {% endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - name: job-exit-spec-rest-server configMap: name: job-exit-spec-configuration +{%- endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} {%- if cluster_cfg['hivedscheduler']['config']|length > 1 %} - name: hived-spec-rest-server configMap: name: hivedscheduler-config {%- endif %} -{%- if cluster_cfg['rest-server']['launcher-type'] == "k8s" %} - name: k8s-exit-spec-rest-server configMap: name: k8s-job-exit-spec-configuration diff --git a/src/rest-server/deploy/stop.sh b/src/rest-server/deploy/stop.sh index 9b679fb426..0e2333a841 100644 --- a/src/rest-server/deploy/stop.sh +++ b/src/rest-server/deploy/stop.sh @@ -27,7 +27,7 @@ if kubectl get configmap | grep -q "auth-configuration"; then kubectl delete configmap auth-configuration || exit $? fi -if kubectl get configmap | grep -q "job-exit-spec-configuration"; then +if kubectl get configmap | grep -q " job-exit-spec-configuration"; then kubectl delete configmap job-exit-spec-configuration || exit $? fi diff --git a/src/rest-server/src/config/yarn.js b/src/rest-server/src/config/yarn.js index d153217d85..b4598ad04b 100644 --- a/src/rest-server/src/config/yarn.js +++ b/src/rest-server/src/config/yarn.js @@ -18,6 +18,7 @@ // module dependencies const Joi = require('joi'); +const launcherConfig = require('@pai/config/launcher'); // get config from environment variables let yarnConfig = { @@ -53,11 +54,12 @@ const yarnConfigSchema = Joi.object().keys({ .required(), }).required(); -const {error, value} = Joi.validate(yarnConfig, yarnConfigSchema); -if (error) { - throw new Error(`yarn config error\n${error}`); +if (launcherConfig.type === 'yarn') { + const {error, value} = Joi.validate(yarnConfig, yarnConfigSchema); + if (error) { + throw new Error(`yarn config error\n${error}`); + } + yarnConfig = value; } -yarnConfig = value; - module.exports = yarnConfig; diff --git a/src/storage-manager/config/storage-manager.yaml b/src/storage-manager/config/storage-manager.yaml index 3037bee3d0..b58de72f70 100644 --- a/src/storage-manager/config/storage-manager.yaml +++ b/src/storage-manager/config/storage-manager.yaml @@ -15,6 +15,8 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "common" + # Global config localpath: /share diff --git a/src/webportal/config/webportal.md b/src/webportal/config/webportal.md index 806f3541d1..44e434b59b 100644 --- a/src/webportal/config/webportal.md +++ b/src/webportal/config/webportal.md @@ -18,7 +18,6 @@ For example, if you want to use different port than the default 9286, add follow ```yaml webportal: server-port: new-value - log-type: yarn ``` About config the web portal plugin, see [PLUGINS.md](../../../docs/webportal/PLUGINS.md) @@ -30,7 +29,6 @@ After parsing, object model looks like: ```yaml webportal: server-port: 9286 - log-type: yarn uri: "http://master_ip:9286" ``` @@ -49,12 +47,6 @@ webportal: cluster_cfg["webportal"]["server-port"] Int - - webportal.log-type - com["webportal"]["log-type"] - cluster_cfg["webportal"]["log-type"] - String - webportal.uri com["webportal"]["uri"] diff --git a/src/webportal/config/webportal.py b/src/webportal/config/webportal.py index 67862d2cf2..6b604f067e 100644 --- a/src/webportal/config/webportal.py +++ b/src/webportal/config/webportal.py @@ -35,8 +35,6 @@ def validation_pre(self): machine_list = self.cluster_configuration['machine-list'] if len([host for host in machine_list if host.get('pai-master') == 'true']) != 1: return False, '1 and only 1 "pai-master=true" machine is required to deploy the rest server' - if self.service_configuration['log-type'] not in ['yarn', 'log-manager']: - return False, '"log-type" should be yarn or log-manager' return True, None #### Generate the final service object model @@ -58,12 +56,10 @@ def apply_config(plugin): machine_list = self.cluster_configuration['machine-list'] master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0] server_port = self.service_configuration['server-port'] - log_type = self.service_configuration['log-type'] uri = 'http://{0}:{1}'.format(master_ip, server_port) plugins = self.service_configuration['plugins'] return { 'server-port': server_port, - 'log-type': log_type, 'uri': uri, 'plugins': json.dumps([apply_config(plugin) for plugin in plugins]), 'webportal-address': master_ip, @@ -71,16 +67,18 @@ def apply_config(plugin): #### All service and main module (kubrenetes, machine) is generated. And in this check steps, you could refer to the service object model which you will used in your own service, and check its existence and correctness. def validation_post(self, cluster_object_model): - for (service, config) in ( + check_tuple = ( ('rest-server', 'uri'), ('prometheus', 'url'), - ('hadoop-resource-manager', 'master-ip'), ('grafana', 'url'), # TODO - #('kubernetes', 'dashboard-url'), + # ('kubernetes', 'dashboard-url'), ('node-exporter', 'port'), ('prometheus', 'scrape_interval'), - ): + ) + if cluster_object_model['cluster']['common']['cluster-type'] == 'yarn': + check_tuple = (('hadoop-resource-manager', 'master-ip'),)+check_tuple + for (service, config) in check_tuple: if service not in cluster_object_model or config not in cluster_object_model[service]: return False, '{0}.{1} is required'.format(service, config) diff --git a/src/webportal/config/webportal.yaml b/src/webportal/config/webportal.yaml index 5cfc657d00..490002d545 100644 --- a/src/webportal/config/webportal.yaml +++ b/src/webportal/config/webportal.yaml @@ -15,5 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "common" + server-port: 9286 -log-type: yarn \ No newline at end of file diff --git a/src/webportal/deploy/webportal.yaml.template b/src/webportal/deploy/webportal.yaml.template index fb6f4e9fd7..5b1224c140 100644 --- a/src/webportal/deploy/webportal.yaml.template +++ b/src/webportal/deploy/webportal.yaml.template @@ -37,15 +37,21 @@ spec: imagePullPolicy: Always env: - name: LAUNCHER_TYPE - value: {{ cluster_cfg['rest-server']['launcher-type'] }} + value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }} - name: REST_SERVER_URI value: {{ cluster_cfg['rest-server']['uri'] }} - name: PROMETHEUS_URI value: {{ cluster_cfg['prometheus']['url'] }} - name: LOG_TYPE - value: {{ cluster_cfg['webportal']['log-type'] }} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "k8s" %} + value: log-manager +{%- else %} + value: yarn +{%- endif %} +{%- if cluster_cfg["cluster"]["common"]["cluster-type"] == "yarn" %} - name: YARN_WEB_PORTAL_URI value: http://{{ cluster_cfg['hadoop-resource-manager']['master-ip'] }}:8088 +{%- endif %} - name: GRAFANA_URI value: {{ cluster_cfg['grafana']['url'] }} - name: K8S_DASHBOARD_URI diff --git a/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml b/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml index 2348238d2e..a25baac66a 100644 --- a/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml +++ b/src/yarn-frameworklauncher/config/yarn-frameworklauncher.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" + frameworklauncher-port: 9086 \ No newline at end of file diff --git a/src/zookeeper/config/zookeeper.yaml b/src/zookeeper/config/zookeeper.yaml index 081aeaac6b..e2cf5810b9 100644 --- a/src/zookeeper/config/zookeeper.yaml +++ b/src/zookeeper/config/zookeeper.yaml @@ -15,4 +15,6 @@ # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +service_type: "yarn" +