Skip to content

Commit

Permalink
feat(helm): make it possbile to install universal cp on k8s (#5913)
Browse files Browse the repository at this point in the history
* feat(helm): make it possbile to install universal cp on k8s

Signed-off-by: slonka <slonka@users.noreply.github.com>
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
  • Loading branch information
slonka and github-actions[bot] authored Feb 6, 2023
1 parent 59ee5f9 commit 33b69bf
Show file tree
Hide file tree
Showing 42 changed files with 766 additions and 50 deletions.
23 changes: 23 additions & 0 deletions app/kumactl/cmd/install/install_control_plane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,19 @@ var _ = Describe("kumactl install control-plane", func() {
},
goldenFile: "install-control-plane.global.golden.yaml",
}),
Entry("should generate Kubernetes resources for Global Universal mode", testCase{
extraArgs: []string{
"--mode",
"global",
"--set",
"controlPlane.environment=universal",
"--set",
"postgres.tls.mode=verifyFull",
"--set",
"postgres.tls.secretName=postgres-tls-secret-name",
},
goldenFile: "install-control-plane.global-universal-on-k8s.golden.yaml",
}),
Entry("should generate Kubernetes resources for Zone", testCase{
extraArgs: []string{
"--mode", "zone",
Expand Down Expand Up @@ -303,6 +316,16 @@ controlPlane:
extraArgs: []string{"--mode", "test"},
errorMsg: "controlPlane.mode invalid got:'test'",
}),
Entry("--mode is not global and environment is universal", errTestCase{
extraArgs: []string{
"--mode",
"zone",
"--set",
"controlPlane.environment=universal",
},
errorMsg: "Currently you can only run universal mode on kubernetes in a global mode, " +
"this limitation might be lifted in the future",
}),
Entry("--kds-global-address is missing when installing zone", errTestCase{
extraArgs: []string{"--mode", "zone", "--zone", "zone-1"},
errorMsg: "controlPlane.kdsGlobalAddress can't be empty when controlPlane.mode=='zone'",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 86ea761718bf0f0e59a68251ac0eace6afa0a6f256acc3f738729980f77e310d
checksum/tls-secrets: 8fc117aa00b995fc1ec633b9f8f2be15e7f7c325460ce6c06bed414fd4bdc8e4
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 86ea761718bf0f0e59a68251ac0eace6afa0a6f256acc3f738729980f77e310d
checksum/tls-secrets: 8fc117aa00b995fc1ec633b9f8f2be15e7f7c325460ce6c06bed414fd4bdc8e4
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5229,7 +5229,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 86ea761718bf0f0e59a68251ac0eace6afa0a6f256acc3f738729980f77e310d
checksum/tls-secrets: 8fc117aa00b995fc1ec633b9f8f2be15e7f7c325460ce6c06bed414fd4bdc8e4
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ installCrdsOnUpgrade:
imagePullSecrets: []

controlPlane:
# -- Environment that control plane is run in, useful when running universal global control plane on k8s
environment: "kubernetes"

# -- Labels to add to resources in addition to default labels
extraLabels: {}

Expand Down Expand Up @@ -699,6 +702,32 @@ experimental:
# -- Path where compiled eBPF programs which will be installed can be found
programsSourcePath: /kuma/ebpf

# Postgres' settings for universal control plane on k8s
postgres:
# -- Postgres hostname
host:
# -- Postgres port
port: "5432"
# -- Postgres user, password should be provided as a secret reference in "controlPlane.secrets"
# with the Env value "KUMA_STORE_POSTGRES_PASSWORD".
# Example:
# controlPlane:
# secrets:
# - Secret: postgres-postgresql
# Key: postgresql-password
# Env: KUMA_STORE_POSTGRES_PASSWORD
user:
# -- Postgres database to use
db:
# TLS settings
tls:
# -- Mode of TLS connection. Available values are: "disable", "verifyNone", "verifyCa", "verifyFull"
mode: disable # ENV: KUMA_STORE_POSTGRES_TLS_MODE
# -- Whether to disable SNI the postgres `sslsni` option.
disableSSLSNI: false # ENV: KUMA_STORE_POSTGRES_TLS_DISABLE_SSLSNI
# -- Secret that contains tls.crt, tls.key, rootCA.crt
secretName:

# @ignored for helm-docs
plugins:
policies:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5229,7 +5229,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 86ea761718bf0f0e59a68251ac0eace6afa0a6f256acc3f738729980f77e310d
checksum/tls-secrets: 8fc117aa00b995fc1ec633b9f8f2be15e7f7c325460ce6c06bed414fd4bdc8e4
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5383,7 +5383,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 86ea761718bf0f0e59a68251ac0eace6afa0a6f256acc3f738729980f77e310d
checksum/tls-secrets: 8fc117aa00b995fc1ec633b9f8f2be15e7f7c325460ce6c06bed414fd4bdc8e4
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@

---
apiVersion: v1
kind: Namespace
metadata:
name: kuma-system
labels:
kuma.io/system-namespace: "true"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kuma-control-plane
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kuma-control-plane-config
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
data:
config.yaml: |
# use this file to override default configuration of `kuma-cp`
#
# see conf/kuma-cp.conf.yml for available settings
---
apiVersion: v1
kind: Service
metadata:
name: kuma-global-zone-sync
namespace: kuma-system
annotations:
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
spec:
type: LoadBalancer
ports:
- port: 5685
name: global-zone-sync
selector:
app: kuma-control-plane

app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
---
apiVersion: v1
kind: Service
metadata:
name: kuma-control-plane
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5680"
spec:
type: ClusterIP
ports:
- port: 5680
name: diagnostics
- port: 5681
name: http-api-server
- port: 5682
name: https-api-server
selector:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kuma-control-plane
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
spec:
replicas: 1
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
selector:
matchLabels:
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
app: kuma-control-plane
template:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- 'kuma'
- key: app.kubernetes.io/instance
operator: In
values:
- 'kuma'
- key: app
operator: In
values:
- 'kuma-control-plane'
topologyKey: kubernetes.io/hostname
weight: 100
serviceAccountName: kuma-control-plane
automountServiceAccountToken: true
nodeSelector:

kubernetes.io/os: linux
hostNetwork: false
terminationGracePeriodSeconds: 30
initContainers:
- name: migration
image: "docker.io/kumahq/kuma-cp:0.0.1"
imagePullPolicy: IfNotPresent
env:
- name: KUMA_DEFAULTS_SKIP_MESH_CREATION
value: "false"
- name: KUMA_ENVIRONMENT
value: "universal"
- name: KUMA_GENERAL_WORK_DIR
value: "/tmp/kuma"
- name: KUMA_MODE
value: "global"
- name: KUMA_STORE_POSTGRES_DB_NAME
value: ""
- name: KUMA_STORE_POSTGRES_HOST
value: ""
- name: KUMA_STORE_POSTGRES_PORT
value: "5432"
- name: KUMA_STORE_POSTGRES_TLS_CERT_PATH
value: "/var/run/secrets/kuma.io/postgres-client-certs/tls.crt"
- name: KUMA_STORE_POSTGRES_TLS_KEY_PATH
value: "/var/run/secrets/kuma.io/postgres-client-certs/tls.key"
- name: KUMA_STORE_POSTGRES_TLS_MODE
value: "verifyFull"
- name: KUMA_STORE_POSTGRES_TLS_ROOT_CERT_PATH
value: "/var/run/secrets/kuma.io/postgres-client-certs/rootCA.crt"
- name: KUMA_STORE_POSTGRES_USER
value: ""
- name: KUMA_STORE_TYPE
value: "postgres"
args:
- migrate
- up
- --log-level=info
- --config-file=/etc/kuma.io/kuma-control-plane/config.yaml
volumeMounts:
- name: kuma-control-plane-config
mountPath: /etc/kuma.io/kuma-control-plane
readOnly: true
containers:
- name: control-plane
image: "docker.io/kumahq/kuma-cp:0.0.1"
imagePullPolicy: IfNotPresent
env:
- name: KUMA_DEFAULTS_SKIP_MESH_CREATION
value: "false"
- name: KUMA_ENVIRONMENT
value: "universal"
- name: KUMA_GENERAL_WORK_DIR
value: "/tmp/kuma"
- name: KUMA_MODE
value: "global"
- name: KUMA_STORE_POSTGRES_DB_NAME
value: ""
- name: KUMA_STORE_POSTGRES_HOST
value: ""
- name: KUMA_STORE_POSTGRES_PORT
value: "5432"
- name: KUMA_STORE_POSTGRES_TLS_CERT_PATH
value: "/var/run/secrets/kuma.io/postgres-client-certs/tls.crt"
- name: KUMA_STORE_POSTGRES_TLS_KEY_PATH
value: "/var/run/secrets/kuma.io/postgres-client-certs/tls.key"
- name: KUMA_STORE_POSTGRES_TLS_MODE
value: "verifyFull"
- name: KUMA_STORE_POSTGRES_TLS_ROOT_CERT_PATH
value: "/var/run/secrets/kuma.io/postgres-client-certs/rootCA.crt"
- name: KUMA_STORE_POSTGRES_USER
value: ""
- name: KUMA_STORE_TYPE
value: "postgres"
- name: KUMA_INTER_CP_CATALOG_INSTANCE_ADDRESS
valueFrom:
fieldRef:
fieldPath: status.podIP
args:
- run
- --log-level=info
- --config-file=/etc/kuma.io/kuma-control-plane/config.yaml
ports:
- containerPort: 5680
name: diagnostics
protocol: TCP
- containerPort: 5681
- containerPort: 5682
- containerPort: 5443
livenessProbe:
timeoutSeconds: 10
httpGet:
path: /healthy
port: 5680
readinessProbe:
timeoutSeconds: 10
httpGet:
path: /ready
port: 5680
resources:
requests:
cpu: 500m
memory: 256Mi

volumeMounts:
- name: kuma-control-plane-config
mountPath: /etc/kuma.io/kuma-control-plane
readOnly: true
- name: postgres-client-certs
mountPath: /var/run/secrets/kuma.io/postgres-client-certs
readOnly: true
volumes:
- name: postgres-client-certs
secret:
secretName: postgres-tls-secret-name
- name: kuma-control-plane-config
configMap:
name: kuma-control-plane-config
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 7f15a049cce92f3c965fd34e12471da804ed4ccf2cc861e2c7b0f0dfa7a7115c
checksum/tls-secrets: 6a8a5ebea10cd0614847f009070d60475d7e6ca0338412c019ac1fcd85b368f1
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ spec:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 86ea761718bf0f0e59a68251ac0eace6afa0a6f256acc3f738729980f77e310d
checksum/tls-secrets: 8fc117aa00b995fc1ec633b9f8f2be15e7f7c325460ce6c06bed414fd4bdc8e4
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
Expand Down
Loading

0 comments on commit 33b69bf

Please sign in to comment.