From e85f8a994bf3ce98f8e0353273a4622542c0b796 Mon Sep 17 00:00:00 2001 From: yanghua Date: Tue, 7 Feb 2023 18:16:42 +0800 Subject: [PATCH] [Addon #603] Add Apache Spark as a experimental addon Signed-off-by: yanghua --- addons/spark-kubernetes-operator/README.md | 3 + .../spark-kubernetes-operator/metadata.yaml | 15 +++++ .../spark-kubernetes-operator/parameter.cue | 22 +++++++ .../schemas/myschema.yaml | 8 +++ addons/spark-kubernetes-operator/template.cue | 63 +++++++++++++++++++ .../views/my-view.cue | 31 +++++++++ 6 files changed, 142 insertions(+) create mode 100644 addons/spark-kubernetes-operator/README.md create mode 100644 addons/spark-kubernetes-operator/metadata.yaml create mode 100644 addons/spark-kubernetes-operator/parameter.cue create mode 100644 addons/spark-kubernetes-operator/schemas/myschema.yaml create mode 100644 addons/spark-kubernetes-operator/template.cue create mode 100644 addons/spark-kubernetes-operator/views/my-view.cue diff --git a/addons/spark-kubernetes-operator/README.md b/addons/spark-kubernetes-operator/README.md new file mode 100644 index 000000000..1872201f0 --- /dev/null +++ b/addons/spark-kubernetes-operator/README.md @@ -0,0 +1,3 @@ +# spark-kubernetes-operator + +This is an addon template. Check how to build your own addon: https://kubevela.net/docs/platform-engineers/addon/intro diff --git a/addons/spark-kubernetes-operator/metadata.yaml b/addons/spark-kubernetes-operator/metadata.yaml new file mode 100644 index 000000000..485fe4286 --- /dev/null +++ b/addons/spark-kubernetes-operator/metadata.yaml @@ -0,0 +1,15 @@ +description: A kubernetes operator for Apache Spark +icon: "https://spark.apache.org/images/spark-logo.png" +invisible: false +name: spark-kubernetes-operator +tags: + - GoogleCloudPlatform/spark-on-k8s-operator +version: v1beta2-1.3.8-3.1.1 +url: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator + +dependencies: + - name: fluxcd + +system: + vela: ">=1.5.0-beta.3" +# kubernetes: ">=1.16" diff --git a/addons/spark-kubernetes-operator/parameter.cue b/addons/spark-kubernetes-operator/parameter.cue new file mode 100644 index 000000000..7916c0afc --- /dev/null +++ b/addons/spark-kubernetes-operator/parameter.cue @@ -0,0 +1,22 @@ +// parameter.cue is used to store addon parameters. +// +// You can use these parameters in template.cue or in resources/ by 'parameter.myparam' +// +// For example, you can use parameters to allow the user to customize +// container images, ports, and etc. +parameter: { + // +usage=Custom parameter description + myparam: *"myns" | string + //+usage=Deploy to specified clusters. Leave empty to deploy to all clusters. + clusters?: [...string] + //+usage=Namespace to deploy to, defaults to cert-manager + namespace: *"spark-operator" | string + // +usage=Specify if create the webhook or not + "createWebhook": *true | bool + // +usage=Specify the image repository + "imageRepository": *"ghcr.io/googlecloudplatform/spark-operator" | string + // +usage=Specify the image tag + "imageTag": *"v1beta2-1.3.8-3.1.1" | string + // +usage=Specify if create the sa for job or not + "createSparkServiceAccount": *false|bool +} diff --git a/addons/spark-kubernetes-operator/schemas/myschema.yaml b/addons/spark-kubernetes-operator/schemas/myschema.yaml new file mode 100644 index 000000000..8f5d801c8 --- /dev/null +++ b/addons/spark-kubernetes-operator/schemas/myschema.yaml @@ -0,0 +1,8 @@ +# We put UI Schemas that correspond to Definitions in schemas directory. +# References: +# - https://kubevela.net/docs/platform-engineers/addon/intro#schemas-directoryoptional +# - https://kubevela.net/docs/reference/ui-schema +- jsonKey: myparam + label: MyParam + validate: + required: true diff --git a/addons/spark-kubernetes-operator/template.cue b/addons/spark-kubernetes-operator/template.cue new file mode 100644 index 000000000..69823f82a --- /dev/null +++ b/addons/spark-kubernetes-operator/template.cue @@ -0,0 +1,63 @@ +package main +output: { + apiVersion: "core.oam.dev/v1beta1" + kind: "Application" + spec: { + components: [ + { + type: "k8s-objects" + name: "spark-operator-ns" + properties: objects: [{ + apiVersion: "v1" + kind: "Namespace" + metadata: name: parameter.namespace + }] + }, + { + name: "spark-operator-helm" + type: "helm" + dependsOn: ["spark-operator-ns"] + type: "helm" + properties: { + repoType: "helm" + url: "https://googlecloudplatform.github.io/spark-on-k8s-operator/" + chart: "spark-operator" + targetNamespace: parameter["namespace"] + version: "1.1.26" + values: { + image: { + repository: parameter["imageRepository"] + tag: parameter["imageTag"] + } + + serviceAccounts: { + spark: { + create: parameter["createSparkServiceAccount"] + } + } + + serviceAccounts: { + sparkoperator: { + name: "spark-kubernetes-operator" + } + } + + webhook: { + enable: parameter["createWebhook"] + } + + } + } + }, + ] + policies: [ + { + name: "gc-dependency", + type: "garbage-collect", + properties: { + order: "dependency" + } + } + ] + } +} diff --git a/addons/spark-kubernetes-operator/views/my-view.cue b/addons/spark-kubernetes-operator/views/my-view.cue new file mode 100644 index 000000000..202b2114d --- /dev/null +++ b/addons/spark-kubernetes-operator/views/my-view.cue @@ -0,0 +1,31 @@ +// We put VelaQL views in views directory. +// +// VelaQL(Vela Query Language) is a resource query language for KubeVela, +// used to query status of any extended resources in application-level. +// Reference: https://kubevela.net/docs/platform-engineers/system-operation/velaql +// +// This VelaQL View querys the status of this addon. +// Use this view to query by: +// vela ql --query 'my-view{addonName:spark-kubernetes-operator}.status' +// You should see 'running'. + +import ( + "vela/ql" +) + +app: ql.#Read & { + value: { + kind: "Application" + apiVersion: "core.oam.dev/v1beta1" + metadata: { + name: "addon-" + parameter.addonName + namespace: "vela-system" + } + } +} + +parameter: { + addonName: *"spark-kubernetes-operator" | string +} + +status: app.value.status.status