Create role and rolebinding for controller/webhook in every spark job…

… namespace if not watching all namespaces (#2129) watching all namespaces Signed-off-by: Yi Chen <github@chenyicn.net>
kubeflow · Sep 8, 2024 · 592b649 · 592b649
1 parent 62b4ca6
commit 592b649
Show file tree

Hide file tree

Showing 8 changed files with 546 additions and 149 deletions.
diff --git a/charts/spark-operator-chart/templates/controller/_helpers.tpl b/charts/spark-operator-chart/templates/controller/_helpers.tpl
@@ -48,6 +48,34 @@ Create the name of the service account to be used by the controller
 {{- end -}}
 {{- end -}}
 
+{{/*
+Create the name of the cluster role to be used by the controller
+*/}}
+{{- define "spark-operator.controller.clusterRoleName" -}}
+{{ include "spark-operator.controller.name" . }}
+{{- end }}
+
+{{/*
+Create the name of the cluster role binding to be used by the controller
+*/}}
+{{- define "spark-operator.controller.clusterRoleBindingName" -}}
+{{ include "spark-operator.controller.clusterRoleName" . }}
+{{- end }}
+
+{{/*
+Create the name of the role to be used by the controller
+*/}}
+{{- define "spark-operator.controller.roleName" -}}
+{{ include "spark-operator.controller.name" . }}
+{{- end }}
+
+{{/*
+Create the name of the role binding to be used by the controller
+*/}}
+{{- define "spark-operator.controller.roleBindingName" -}}
+{{ include "spark-operator.controller.roleName" . }}
+{{- end }}
+
 {{/*
 Create the name of the deployment to be used by controller
 */}}
@@ -68,3 +96,88 @@ Create the name of the pod disruption budget to be used by controller
 {{- define "spark-operator.controller.podDisruptionBudgetName" -}}
 {{ include "spark-operator.controller.name" . }}-pdb
 {{- end -}}
+
+{{/*
+Create the role policy rules for the controller in every Spark job namespace
+*/}}
+{{- define "spark-operator.controller.policyRules" -}}
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - extensions
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - create
+  - delete
+- apiGroups:
+  - sparkoperator.k8s.io
+  resources:
+  - sparkapplications
+  - scheduledsparkapplications
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - sparkoperator.k8s.io
+  resources:
+  - sparkapplications/status
+  - sparkapplications/finalizers
+  - scheduledsparkapplications/status
+  - scheduledsparkapplications/finalizers
+  verbs:
+  - get
+  - update
+  - patch
+{{- if .Values.controller.batchScheduler.enable }}
+{{/* required for the `volcano` batch scheduler */}}
+- apiGroups:
+  - scheduling.incubator.k8s.io
+  - scheduling.sigs.dev
+  - scheduling.volcano.sh
+  resources:
+  - podgroups
+  verbs:
+  - "*"
+{{- end }}
+{{- end -}}
diff --git a/charts/spark-operator-chart/templates/controller/rbac.yaml b/charts/spark-operator-chart/templates/controller/rbac.yaml
@@ -18,58 +18,15 @@ limitations under the License.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ include "spark-operator.controller.name" . }}
+  name: {{ include "spark-operator.controller.clusterRoleName" . }}
+  namespace: {{ .Release.Namespace }}
   labels:
     {{- include "spark-operator.controller.labels" . | nindent 4 }}
   {{- with .Values.controller.rbac.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
-  - deletecollection
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - get
-  - create
-  - update
-  - patch
-  - delete
-- apiGroups:
-  - ""
-  resources:
-  - services
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
-- apiGroups:
-  - extensions
-  - networking.k8s.io
-  resources:
-  - ingresses
-  verbs:
-  - get
-  - create
-  - delete
 - apiGroups:
   - ""
   resources:
@@ -90,63 +47,16 @@ rules:
   - customresourcedefinitions
   verbs:
   - get
-- apiGroups:
-  - sparkoperator.k8s.io
-  resources:
-  - sparkapplications
-  - scheduledsparkapplications
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - patch
-  - delete
-- apiGroups:
-  - sparkoperator.k8s.io
-  resources:
-  - sparkapplications/status
-  - scheduledsparkapplications/status
-  verbs:
-  - get
-  - update
-  - patch
-- apiGroups:
-  - sparkoperator.k8s.io
-  resources:
-  - sparkapplications/finalizers
-  - scheduledsparkapplications/finalizers
-  verbs:
-  - update
-{{- if .Values.controller.batchScheduler.enable }}
-{{/* required for the `volcano` batch scheduler */}}
-- apiGroups:
-  - scheduling.incubator.k8s.io
-  - scheduling.sigs.dev
-  - scheduling.volcano.sh
-  resources:
-  - podgroups
-  verbs:
-  - "*"
-- apiGroups:
-  - scheduling.x-k8s.io
-  resources:
-  - podgroups
-  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - delete
+{{- if not .Values.spark.jobNamespaces | or (has "" .Values.spark.jobNamespaces) }}
+{{ include "spark-operator.controller.policyRules" . }}
 {{- end }}
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ include "spark-operator.controller.name" . }}
+  name: {{ include "spark-operator.controller.clusterRoleBindingName" . }}
+  namespace: {{ .Release.Namespace }}
   labels:
     {{- include "spark-operator.controller.labels" . | nindent 4 }}
   {{- with .Values.controller.rbac.annotations }}
@@ -160,13 +70,14 @@ subjects:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: {{ include "spark-operator.controller.name" . }}
-
+  name: {{ include "spark-operator.controller.clusterRoleName" . }}
 ---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ include "spark-operator.controller.name" . }}
+  name: {{ include "spark-operator.controller.roleName" . }}
+  namespace: {{ .Release.Namespace }}
   labels:
     {{- include "spark-operator.controller.labels" . | nindent 4 }}
   {{- with .Values.controller.rbac.annotations }}
@@ -189,12 +100,16 @@ rules:
   verbs:
   - get
   - update
-
+{{- if has .Release.Namespace .Values.spark.jobNamespaces }}
+{{ include "spark-operator.controller.policyRules" . }}
+{{- end }}
 ---
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ include "spark-operator.controller.name" . }}
+  name: {{ include "spark-operator.controller.roleBindingName" . }}
+  namespace: {{ .Release.Namespace }}
   labels:
     {{- include "spark-operator.controller.labels" . | nindent 4 }}
   {{- with .Values.controller.rbac.annotations }}
@@ -208,5 +123,48 @@ subjects:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: {{ include "spark-operator.controller.name" . }}
+  name: {{ include "spark-operator.controller.roleName" . }}
+
+{{- if and .Values.spark.jobNamespaces (not (has "" .Values.spark.jobNamespaces)) }}
+{{- range $jobNamespace := .Values.spark.jobNamespaces }}
+{{- if ne $jobNamespace $.Release.Namespace }}
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "spark-operator.controller.roleName" $ }}
+  namespace: {{ $jobNamespace }}
+  labels:
+    {{- include "spark-operator.controller.labels" $ | nindent 4 }}
+  {{- with $.Values.controller.rbac.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+rules:
+{{ include "spark-operator.controller.policyRules" $ }}
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "spark-operator.controller.roleBindingName" $ }}
+  namespace: {{ $jobNamespace }}
+  labels:
+    {{- include "spark-operator.controller.labels" $ | nindent 4 }}
+  {{- with $.Values.controller.rbac.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "spark-operator.controller.serviceAccountName" $ }}
+  namespace: {{ $.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "spark-operator.controller.roleName" $ }}
+{{- end }}
+{{- end }}
+{{- end }}
 {{- end }}
diff --git a/charts/spark-operator-chart/templates/controller/serviceaccount.yaml b/charts/spark-operator-chart/templates/controller/serviceaccount.yaml
@@ -19,6 +19,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{ include "spark-operator.controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
   labels:
     {{- include "spark-operator.controller.labels" . | nindent 4 }}
   {{- with .Values.controller.serviceAccount.annotations }}