[Cluster Object Model] Generate necessary service config based on clu…

…ster type (#4036)

* init commit

* [Cluster Object Model] Check laucher-type and log-type based on cluster-type. (#4038)

* Remove .k8s.yaml logic. And remove launcher-type & log-type

* [Cluster Objec Model]Don't check hadoop rm uri when in pure k8s cluster. (#4052)

* [Rest-server] Add env, volume based on the cluster-type (#4058)

* Remove yarn config from pylon when cluster-type is k8s (#4063)

* [cluster object model] Document update. (#4066)

* From openpai_parser_type  to service_type

deployment/clusterObjectModel/cluster_object_model.py

@@ -54,20 +54,25 @@ def get_service_model_list(self):
     def get_service_parser(self, service_name, cluster_type):
 
         sys.path.insert(0, '{0}/../../src/{1}/config'.format(package_directory_com, service_name))
-        default_path = "{0}/../../src/{1}/config/{1}.{2}.yaml".format(package_directory_com, service_name, cluster_type)
-        if not file_handler.file_exist_or_not(default_path):
-            default_path = "{0}/../../src/{1}/config/{1}.yaml".format(package_directory_com, service_name)
+        default_path = "{0}/../../src/{1}/config/{1}.yaml".format(package_directory_com, service_name)
 
         # Prepare Service Configuration
         layout = self.layout
-
-        default_service_cfg = []
+        service_type = "common"
+        default_service_cfg = {}
         if file_handler.file_exist_or_not(default_path):
             default_service_cfg = file_handler.load_yaml_config(default_path)
+        if default_service_cfg is not None and "service_type" in default_service_cfg:
+            service_type = default_service_cfg["service_type"]
 
         overwrite_service_cfg = {}
         if self.overwrite_service_configuration is not None and service_name in self.overwrite_service_configuration:
             overwrite_service_cfg = self.overwrite_service_configuration[service_name]
+        if "service_type" in overwrite_service_cfg:
+            service_type = overwrite_service_cfg["service_type"]
+
+        if service_type != "common" and service_type != cluster_type:
+            return None
 
         # Init parser instance
         parser_module = importlib.import_module(service_name.replace("-", "_"))
@@ -128,7 +133,11 @@ def service_config(self):
 
         service_model_list = self.get_service_model_list()
         for service_name in service_model_list:
-            parser_dict[service_name] = self.get_service_parser(service_name, cluster_type)
+            parser = self.get_service_parser(service_name, cluster_type)
+            if parser is None:
+                continue
+            parser_dict[service_name] = parser
+
         return self.load_config(parser_dict)
 
     def kubernetes_config(self):

deployment/clusterObjectModel/test/test_template_generate.py

@@ -67,11 +67,12 @@ def test_template_generate(self):
 
         subdir_list = directory_handler.get_subdirectory_list(src_path)
         for subdir in subdir_list:
-
             service_deploy_dir = "{0}/{1}/deploy".format(src_path, subdir)
-            service_deploy_conf_path =  "{0}/{1}/deploy/service.yaml".format(src_path, subdir)
+            service_deploy_conf_path = "{0}/{1}/deploy/service.yaml".format(src_path, subdir)
             if file_handler.directory_exits(service_deploy_dir) and file_handler.file_exist_or_not(service_deploy_conf_path):
-                service_list.append(subdir)
+                service_conf = file_handler.load_yaml_config(service_deploy_conf_path)
+                if ("cluster-type" not in service_conf) or ("cluster-type" in service_conf and "yarn" in service_conf["cluster-type"]):
+                    service_list.append(subdir)
 
         for serv in service_list:
             service_conf = file_handler.load_yaml_config("{0}/{1}/deploy/service.yaml".format(src_path, serv))

deployment/quick-start/services-configuration.yaml.template

@@ -65,8 +65,6 @@ cluster:
 
 
 rest-server:
-#  # launcher type. k8s or yarn
-#  launcher-type: k8s
   # database admin username
   default-pai-admin-username: admin
   # database admin password
@@ -76,12 +74,13 @@ rest-server:
 # uncomment following section if you want to customize the port of web portal
 # webportal:
 #   server-port: 9286
-#   log-type: yarn
+
 
 # uncomment following if you want to change customeize grafana
 # grafana:
 #   port: 3000
 
+
 # uncomment following if you want to change customeize drivers
 #drivers:
 #  set-nvidia-runtime: false

docs/pai-management/doc/how-to-add-new-conm.md

@@ -35,6 +35,14 @@ src/${service_name}/config/${service_name}.yaml
 
 ${service_name}.yaml
 ```yaml
+# The key `service_type` is designed for cluster-object-model. And don't use it in other place.
+# Now, you could configure this key with one of ['yarn', 'common', 'k8s'].
+# When service_type is yarn, the service's configuration will be generated if the cluster-type is yarn.
+# When service_type is k8s, the service's configuration will be generated if the cluster-type is k8s.
+# When service_type is common, the service's configuration will be generated in both of the cluster-type.
+# If the key is missing, default value will be 'common'.
+service_type: "yarn"
+
 # key : value
 service-a-key1: default—value1
 

src/alert-manager/config/alert-manager.yaml

@@ -1,3 +1,22 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
+
 port: 9093
 use-pylon: False
-repeat-interval: '24h'
+repeat-interval: '24h'

src/cleaner/config/cleaner.yaml

@@ -15,5 +15,7 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+
 threshold: 90
 interval: 60

src/cluster/config/cluster.yaml

@@ -15,6 +15,7 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "common"
 
 common:
   cluster-id: pai

src/device-plugin/config/device-plugin.yaml

@@ -15,6 +15,8 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "k8s"
+
 devices:
   - "nvidia.com/gpu"
   - "rdma/hca"

src/drivers/config/drivers.yaml

@@ -15,6 +15,7 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
 
 set-nvidia-runtime: false
 

src/webportal/config/webportal.k8s.yaml → src/elasticsearch/config/elasticsearch.yaml

@@ -15,5 +15,4 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-server-port: 9286
-log-type: log-manager
+service_type: "k8s"

src/grafana/config/grafana.yaml

@@ -1 +1,20 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
+
 port: 3000

src/hadoop-batch-job/config/hadoop-batch-job.yaml

@@ -15,3 +15,4 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"

src/hadoop-data-node/config/hadoop-data-node.yaml

@@ -15,4 +15,6 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+
 storage_path:

src/hadoop-jobhistory/config/hadoop-jobhistory.yaml

@@ -15,3 +15,6 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+
+

src/hadoop-name-node/config/hadoop-name-node.yaml

@@ -15,3 +15,5 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+

src/hadoop-node-manager/config/hadoop-node-manager.yaml

@@ -15,3 +15,5 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+

src/hadoop-resource-manager/config/hadoop-resource-manager.yaml

@@ -15,6 +15,8 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+
 virtualClusters:
   default:
     description: Default VC.

src/hivedscheduler/config/hivedscheduler.yaml

@@ -15,4 +15,8 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+
+service_type: "k8s"
+
 webservice-port: 30096
+

src/internal-storage/config/internal-storage.yaml

@@ -1,3 +1,22 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
+
 enable: false
 type: hostPath
 root-path: /mnt/paiInternal

src/job-exit-spec/config/job-exit-spec.yaml

@@ -15,6 +15,8 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "yarn"
+
 ################################################################################
 # PAI Job ExitSpec Schema
 ################################################################################

src/job-exporter/config/job-exporter.yaml

@@ -1,19 +1,21 @@
 # Copyright (c) Microsoft Corporation
-# # All rights reserved.
-# #
-# # MIT License
-# #
-# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
-# # documentation files (the "Software"), to deal in the Software without restriction, including without limitation
-# # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
-# # to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-# #
-# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
-# # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-# # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
 
 port: 9102
 logging-level: INFO

src/job-exporter/deploy/job-exporter.yaml.template

@@ -71,7 +71,7 @@ spec:
         - name: NVIDIA_VISIBLE_DEVICES
           value: all
         - name: LAUNCHER_TYPE
-          value: {{ cluster_cfg['rest-server']['launcher-type'] }}
+          value: {{ cluster_cfg["cluster"]["common"]["cluster-type"] }}
         {%- if cluster_cfg['cluster']['common']['deploy-in-aks'] == "true" %}
         - name: DEPLOY_ENV
           value: aks

src/k8s-dashboard/config/k8s-dashboard.yaml

@@ -14,3 +14,5 @@
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"

src/k8s-job-exit-spec/config/k8s-job-exit-spec.yaml

@@ -15,6 +15,8 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "k8s"
+
 ################################################################################
 # PAI Job ExitSpec Schema
 ################################################################################

src/log-manager/config/log-manager.yaml

@@ -15,4 +15,6 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+service_type: "k8s"
+
 port: 9103

src/node-exporter/config/node-exporter.yaml

@@ -1,18 +1,20 @@
 # Copyright (c) Microsoft Corporation
-# # All rights reserved.
-# #
-# # MIT License
-# #
-# # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
-# # documentation files (the "Software"), to deal in the Software without restriction, including without limitation
-# # the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
-# # to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-# # The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-# #
-# # THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
-# # BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-# # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
 
 port: 9100

src/postgresql/config/postgresql.yaml

@@ -1,3 +1,22 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+service_type: "common"
+
 enable: false
 user: root
 passwd: rootpass