[Marketplace] Deploy scripts and config (#5066)

* update

* update

* update

* update

* add marketplace-db

* update

* update

* fix

* fix

* add marketplace-restserver

* update

* update

* fix

* update pylon

* update

* update

* update

* update

* update

* update

src/marketplace-db/build/marketplace-db.k8s.dockerfile

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+FROM docker.io/postgres:12.0

src/marketplace-db/config/marketplace-db.yaml

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+service_type: "k8s"
+
+user: user
+passwd: passwd
+db: marketplace
+port: 9291
+max-connection: 1000
+data-path: /mnt/marketplace

src/marketplace-db/config/marketplace_db.py

@@ -0,0 +1,27 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import copy
+
+class MarketplaceDb(object):
+    def __init__(self, cluster_conf, service_conf, default_service_conf):
+        self.cluster_conf = cluster_conf
+        self.service_conf = dict(default_service_conf, **service_conf)
+
+    def validation_pre(self):
+        machine_list = self.cluster_conf['machine-list']
+        if len([host for host in machine_list if host.get('pai-master') == 'true']) < 1:
+            return False, '"pai-master=true" machine is required to deploy the marketplace-db service'
+        return True, None
+
+    def run(self):
+        result = copy.deepcopy(self.service_conf)
+        machine_list = self.cluster_conf['machine-list']
+        master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0]
+        result['host'] = master_ip
+        result['connection-str'] = 'postgresql://{}:{}@{}:{}/{}'.format(
+            result['user'], result['passwd'], result['host'], result['port'], result['db'])
+        return result
+
+    def validation_post(self, conf):
+        return True, None

src/marketplace-db/deploy/delete.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+/bin/bash stop.sh || exit $?
+
+popd > /dev/null

src/marketplace-db/deploy/marketplace-db.yaml.template

@@ -0,0 +1,43 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: marketplace-db-ds
+spec:
+  selector:
+    matchLabels:
+      app: marketplace-db
+  template:
+    metadata:
+      name: marketplace-db
+      labels:
+        app: marketplace-db
+    spec:
+      hostNetwork: true
+      containers:
+      - name: marketplace-db
+        image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}marketplace-db:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
+        imagePullPolicy: Always
+        env:
+        - name: POSTGRES_USER
+          value: {{ cluster_cfg["marketplace-db"]["user"] }}
+        - name: POSTGRES_PASSWORD
+          value: {{ cluster_cfg["marketplace-db"]["passwd"] }}
+        - name: POSTGRES_DB
+          value: {{ cluster_cfg["marketplace-db"]["db"] }}
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        args: ['-c', 'port={{- cluster_cfg["marketplace-db"]["port"] }}', '-N', '{{ cluster_cfg["marketplace-db"]["max-connection"] }}']
+        volumeMounts:
+        - name: marketplace-data-dir
+          mountPath: /var/lib/postgresql/data/pgdata
+          mountPropagation: "None"
+      volumes:
+        - name: marketplace-data-dir
+          hostPath:
+            path: '{{ cluster_cfg["marketplace-db"]["data-path"] }}'
+            type: DirectoryOrCreate
+      imagePullSecrets:
+      - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}

src/marketplace-db/deploy/refresh.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+bash stop.sh
+bash start.sh
+
+popd > /dev/null

src/marketplace-db/deploy/service.yaml

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cluster-type:
+  - k8s
+
+prerequisite:
+  - cluster-configuration
+
+template-list:
+  - marketplace-db.yaml
+  - start.sh
+
+start-script: start.sh
+stop-script: stop.sh
+delete-script: delete.sh
+refresh-script: refresh.sh
+
+deploy-rules:
+  - in: pai-master

src/marketplace-db/deploy/start.sh.template

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+kubectl apply --overwrite=true -f marketplace-db.yaml || exit $?
+
+# Wait until the service is ready.
+PYTHONPATH="../../../deployment" python -m  k8sPaiLibrary.monitorTool.check_pod_ready_status -w -k app -v marketplace-db || exit $?
+
+popd > /dev/null

src/marketplace-db/deploy/stop.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+kubectl delete --ignore-not-found --now "daemonset/marketplace-db-ds"

src/marketplace-restserver/build/marketplace-restserver.k8s.dockerfile

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+FROM docker.io/openpai/pai-marketplace-restserver:v1.2.0

src/marketplace-restserver/config/marketplace-restserver.yaml

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+service_type: "k8s"
+
+db_user: user
+db_password: passwd
+db: marketplace
+# db_host: postgres
+db_port: 9291
+server-port: 9292

src/marketplace-restserver/config/marketplace_restserver.py

@@ -0,0 +1,28 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import copy
+
+class MarketplaceRestserver(object):
+    def __init__(self, cluster_conf, service_conf, default_service_conf):
+        self.cluster_conf = cluster_conf
+        self.service_conf = dict(default_service_conf, **service_conf)
+
+    def validation_pre(self):
+        machine_list = self.cluster_conf['machine-list']
+        if len([host for host in machine_list if host.get('pai-master') == 'true']) < 1:
+            return False, '"pai-master=true" machine is required to deploy the marketplace-restserver service'
+        return True, None
+
+    def run(self):
+        result = copy.deepcopy(self.service_conf)
+        machine_list = self.cluster_conf['machine-list']
+        server_port = self.service_conf['server-port']
+        master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0]
+        result['uri'] = 'http://{0}:{1}'.format(master_ip, server_port)
+        if 'db_host' not in result:
+            result['db_host'] = master_ip
+        return result
+
+    def validation_post(self, conf):
+        return True, None

src/marketplace-restserver/deploy/delete.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+/bin/bash stop.sh || exit $?
+
+popd > /dev/null

src/marketplace-restserver/deploy/marketplace-restserver.yaml.template

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: marketplace-restserver-ds
+spec:
+  selector:
+    matchLabels:
+      app: marketplace-restserver
+  template:
+    metadata:
+      name: marketplace-restserver
+      labels:
+        app: marketplace-restserver
+    spec:
+      hostNetwork: true
+      containers:
+      - name: marketplace-restserver
+        image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}marketplace-restserver:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
+        imagePullPolicy: Always
+        env:
+        - name: DB_USERNAME
+          value: {{ cluster_cfg["marketplace-restserver"]["db_user"] }}
+        - name: DB_PASSWORD
+          value: {{ cluster_cfg["marketplace-restserver"]["db_password"] }}
+        - name: DATABASE
+          value: {{ cluster_cfg["marketplace-restserver"]["db"] }}
+        - name: DB_HOST
+          value: {{ cluster_cfg["marketplace-restserver"]["db_host"] }}
+        - name: DB_PORT
+          value: "{{ cluster_cfg["marketplace-restserver"]["db_port"] }}"
+        - name: NODE_ENV
+          value: production
+        - name: PORT
+          value: "{{ cluster_cfg["marketplace-restserver"]["server-port"] }}"
+      imagePullSecrets:
+      - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}

src/marketplace-restserver/deploy/refresh.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+bash stop.sh
+bash start.sh
+
+popd > /dev/null

src/marketplace-restserver/deploy/service.yaml

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cluster-type:
+  - k8s
+
+prerequisite:
+  - cluster-configuration
+
+template-list:
+  - marketplace-restserver.yaml
+  - start.sh
+
+start-script: start.sh
+stop-script: stop.sh
+delete-script: delete.sh
+refresh-script: refresh.sh
+
+deploy-rules:
+  - in: pai-master

src/marketplace-restserver/deploy/start.sh.template

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+kubectl apply --overwrite=true -f marketplace-restserver.yaml || exit $?
+
+# Wait until the service is ready.
+PYTHONPATH="../../../deployment" python -m  k8sPaiLibrary.monitorTool.check_pod_ready_status -w -k app -v marketplace-restserver || exit $?
+
+popd > /dev/null

src/marketplace-restserver/deploy/stop.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+kubectl delete --ignore-not-found --now "daemonset/marketplace-restserver-ds"

src/marketplace-webportal/build/marketplace-webportal.k8s.dockerfile

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+FROM docker.io/openpai/pai-marketplace-webportal:v1.2.0

src/marketplace-webportal/config/marketplace-webportal.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+service_type: "k8s"
+
+# marketplace_api_uri: marketplace_api_uri
+api-port: 9292
+server-port: 9293

src/marketplace-webportal/config/marketplace_webportal.py

@@ -0,0 +1,29 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import copy
+
+class MarketplaceWebportal(object):
+    def __init__(self, cluster_conf, service_conf, default_service_conf):
+        self.cluster_conf = cluster_conf
+        self.service_conf = dict(default_service_conf, **service_conf)
+
+    def validation_pre(self):
+        machine_list = self.cluster_conf['machine-list']
+        if len([host for host in machine_list if host.get('pai-master') == 'true']) < 1:
+            return False, '"pai-master=true" machine is required to deploy the marketplace-webportal service'
+        return True, None
+
+    def run(self):
+        result = copy.deepcopy(self.service_conf)
+        machine_list = self.cluster_conf['machine-list']
+        server_port = self.service_conf['server-port']
+        api_port = self.service_conf['api-port']
+        master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0]
+        result['uri'] = 'http://{0}:{1}'.format(master_ip, server_port)
+        if 'marketplace_api_uri' not in result:
+            result['marketplace_api_uri'] = 'http://{0}:{1}/api'.format(master_ip, api_port)
+        return result
+
+    def validation_post(self, conf):
+        return True, None

src/marketplace-webportal/deploy/delete.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+/bin/bash stop.sh || exit $?
+
+popd > /dev/null

src/marketplace-webportal/deploy/marketplace-webportal.yaml.template

@@ -0,0 +1,29 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: marketplace-webportal-ds
+spec:
+  selector:
+    matchLabels:
+      app: marketplace-webportal
+  template:
+    metadata:
+      name: marketplace-webportal
+      labels:
+        app: marketplace-webportal
+    spec:
+      hostNetwork: true
+      containers:
+      - name: marketplace-webportal
+        image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}marketplace-webportal:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
+        imagePullPolicy: Always
+        env:
+        - name: MARKETPLACE_API_URL
+          value: {{ cluster_cfg["marketplace-webportal"]["marketplace_api_uri"] }}
+        - name: SERVER_PORT
+          value: "{{ cluster_cfg["marketplace-webportal"]["server-port"] }}"
+      imagePullSecrets:
+      - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}