feat: integrated HA-valkey configuration (#240)

* feat: added vars and defaults for valkey * docs: added valkey config docs * wip * testing local build of GitLab, added docs, pulling in latest valkey w/ HA support * it's closer to working * removed a todo * updated to latest releases * updated gitaly cgroups image * dropped restatement of default values, added privs for cgroups init container * wayne's solution that may work in UDS - didn't work with helm template * so close, registry can't find bucket * fixed typo * fixed registry so it picks up the bucket * no longer use local copy of GitLab * reverted unintended change to valkey package name * fixed whitespace lint error * Apply suggestions from code review Co-authored-by: jacobbmay <134300709+jacobbmay@users.noreply.github.com> Signed-off-by: Joseph Richardson <49208786+JoeHCQ1@users.noreply.github.com> * Update valkey.md Signed-off-by: Joseph Richardson <49208786+JoeHCQ1@users.noreply.github.com> --------- Signed-off-by: Joseph Richardson <49208786+JoeHCQ1@users.noreply.github.com> Co-authored-by: jacobbmay <134300709+jacobbmay@users.noreply.github.com>
defenseunicorns · Dec 5, 2024 · 346eb17 · 346eb17
1 parent 40aa798
commit 346eb17
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 25 deletions.
diff --git a/bundles/rke2/uds-bundle.yaml b/bundles/rke2/uds-bundle.yaml
@@ -28,6 +28,8 @@ x-gitlab-init-security-context: &gitlab-init-security-context
   privileged: true
   readOnlyRootFilesystem: false
   runAsUser: 0
+  allowPrivilegeEscalation: true
+  runAsNonRoot: false
 
 packages:
   # Zarf init
@@ -398,10 +400,11 @@ packages:
   # Gitlab
   - name: gitlab-valkey
     repository: ghcr.io/defenseunicorns/packages/uds/valkey
-    ref: 7.2.6-uds.0-upstream
+    ref: 8.0.1-uds.2-upstream
     overrides:
       valkey:
         uds-valkey-config:
+          namespace: gitlab-valkey
           values:
             - path: custom
               value:
@@ -410,13 +413,34 @@ packages:
                     app.kubernetes.io/name: valkey
                   remoteNamespace: gitlab
                   port: 6379
-                  description: "Ingress from GitLab"
+                  description: "Ingress from GitLab to Valkey read/write ports"
+                - direction: Ingress
+                  selector:
+                    app.kubernetes.io/name: valkey
+                  remoteNamespace: gitlab
+                  port: 26379
+                  description: "Ingress from GitLab to Valkey Sentinel"
             - path: copyPassword
               value:
                 enabled: true
                 namespace: gitlab
                 secretName: gitlab-redis
                 secretKey: password
+          variables:
+            - name: GL_VALKEY_REPLICAS
+              default: 3
+              path: replicas
+        valkey:
+          namespace: gitlab-valkey
+          values:
+            - path: architecture
+              value: replication
+            - path: sentinel.enabled
+              value: true
+            - path: auth.enabled
+              value: true
+            - path: auth.sentinel
+              value: true
 
   - name: gitlab-pgbouncer-rw
     repository: ghcr.io/defenseunicorns/packages/uds/pgbouncer
@@ -519,25 +543,30 @@ packages:
 
   - name: gitlab
     repository: ghcr.io/defenseunicorns/packages/uds/gitlab
-    ref: 17.2.9-uds.0-registry1
+    ref: 17.6.1-uds.0-upstream
     imports:
       - name: GITLAB_DB_USERNAME
         package: gitlab-pgbouncer-rw
+
     overrides:
       gitlab:
         uds-gitlab-config:
           values:
             - path: storage.internal
               value: false
+            - path: storage.createSecret.enabled
+              value: false
             - path: postgres.internal
               value: false
             - path: redis
               value:
                 internal: true
                 selector:
                   app.kubernetes.io/name: valkey
-                namespace: valkey
+                namespace: gitlab-valkey
                 port: 6379
+            - path: redis.sentinel.enabled
+              value: true
             - path: ssh.enabled
               value: true
         uds-gitlab-settings:
@@ -562,7 +591,13 @@ packages:
             - path: gitlab.gitlab-exporter.init.containerSecurityContext
               value: *gitlab-init-security-context
             - path: global.redis.host
-              value: "valkey-master.valkey.svc.cluster.local"
+              value: mymaster
+            - path: global.redis.port
+              value: 6379
+            - path: global.redis.auth.enabled
+              value: true
+            - path: global.redis.sentinelAuth.enabled
+              value: true
             - path: gitlab.gitlab-shell.enabled
               value: true
             # See gitlab's additional manifests, this priority class prevents gitaly from being evicted due to node pressure.
@@ -571,22 +606,7 @@ packages:
             - path: gitlab.gitaly.securityContext.fsGroupChangePolicy  # https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#persistent-volume-permissions
               value: OnRootMismatch
             - path: gitlab.gitaly.cgroups.initContainer.image.tag
-              value: v17.2.9
-            - path: gitlab.gitaly.cgroups.initContainer.securityContext # These should be the default https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/charts/gitlab/charts/gitaly/values.yaml?ref_type=heads#L56
-              value:                                                    # but are overridden somewhere to 1000 so setting them back to 0 here.
-                runAsUser: 0
-                runAsGroup: 0
-                privileged: true  # privileged needs added, was not in upstream helm
-            - path: gitlab.gitaly.containerSecurityContext
-              value:
-                runAsUser: 1000
-                allowPrivilegeEscalation: false
-                runAsNonRoot: true
-                capabilities:
-                  drop: ["ALL"]
-            - path: global.psql.host
-              description: "GitLab DB host should point to kubernetes service for read/write pgbouncer cluster."
-              value: "pgbouncer.gitlab-pgbouncer-rw.svc.cluster.local"
+              value: v17.6.1
           variables:
             - name: MIGRATIONS_RESOURCES
               description: "Gitlab Migrations Resources"
@@ -664,6 +684,15 @@ packages:
               path: global.gitaly.internal.names # Gitlab uses the global top-value to set subcharts: https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/values.yaml?ref_type=heads#L184
               default:
                 - "default"  # Gitlab requires that the first shard always be named `default`: https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html?tab=Helm+chart+%28Kubernetes%29#gitlab-requires-a-default-repository-storage
+            - name: GITLAB_VALKEY_SENTINELS
+              path: global.redis.sentinels
+              default:
+                - host: valkey-node-0.valkey-headless.gitlab-valkey.svc.cluster.local
+                  port: 26379
+                - host: valkey-node-1.valkey-headless.gitlab-valkey.svc.cluster.local
+                  port: 26379
+                - host: valkey-node-2.valkey-headless.gitlab-valkey.svc.cluster.local
+                  port: 26379
             # https://docs.gitlab.com/charts/charts/globals.html#postgresql-load-balancing - configure read replica load balancing hosts list and fine tuning if needed.
             - name: GITLAB_DB_LOAD_BALANCING
               path: global.psql.load_balancing

diff --git a/config/uds-config.yaml b/config/uds-config.yaml
@@ -105,7 +105,9 @@ variables:
     ENDPOINT: "http://replace.with.object.store.url"
     ACCESS_KEY: "replace-me-object-store-access-key"
     SECRET_KEY: "replace-me-object-store-secret-key"
+    REGISTRY_BUCKET: uds-gitlab-registry
   gitlab:
+    GENERATE_STORAGE_SECRET: ""  # This is Zarf for "false" - everything is a string in Zarf.
     GITLAB_DB_NAME: "gitlabdb"
     MIGRATIONS_DB_ENDPOINT: "gitlab-pg.replace.with.db.url"
     TOOLBOX_DB_ENDPOINT: "gitlab-pg.replace.with.db.url"
@@ -114,7 +116,7 @@ variables:
     GITLAB_PAGES_ENABLED: "true"
     GITLAB_SIGNUP_ENABLED: "false"
     GITLAB_BACKUP_SCHEDULE: "0 3 * * *"
-    GITLAB_BACKUP_EXTRA_ARGS: "--skip artifiacts,registry"
+    GITLAB_BACKUP_EXTRA_ARGS: "--skip artifacts,registry"
     BUCKET_SUFFIX: ""
     WEBSERVICE_REPLICAS: 3
     WEBSERVICE_WORKERS: 4

diff --git a/docs/gitlab.md b/docs/gitlab.md
@@ -20,4 +20,4 @@ You can confirm newly created repos are going to different shards by viewing the
 
 ## Prevent Gitaly Shard OOM
 
-The UDS Config variable `GITALY_CGROUPS` allows adminstrators to improve the resiliency of each gitaly node. The proper values are heavily context specific so it is disabled by default. However, the value given to that variable will be substituted directly into the `gitlab.gitaly.cgroups` path [documented here](https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage). Adding these values will reduce the risk a single git call will cause the gitaly node (shard) to be killed with an OOM error.
+The UDS Config variable `GITALY_CGROUPS` allows administrators to improve the resiliency of each gitaly node. The proper values are heavily context specific so it is disabled by default. However, the value given to that variable will be substituted directly into the `gitlab.gitaly.cgroups` path [documented here](https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage). Adding these values will reduce the risk a single git call will cause the gitaly node (shard) to be killed with an OOM error.
diff --git a/docs/valkey.md b/docs/valkey.md
@@ -0,0 +1,5 @@
+# Configuration Notes
+
+## ValKey Clustering
+
+In bundle version 0.5.2 Valkey was changed to deploy replicated by default. For more context, see [this write-up](https://github.com/defenseunicorns/uds-package-valkey/blob/main/docs/configuration.md#high-availability) on how Valkey's HA settings are configured in the UDS Package and [the GitLab Package configuration doc]([TODO](https://github.com/defenseunicorns/uds-package-gitlab/blob/main/docs/configuration.md#valkey-ha-configuration)) to understand the integration with GitLab.
diff --git a/packages/additional-manifests/zarf.yaml b/packages/additional-manifests/zarf.yaml
@@ -21,7 +21,7 @@ components:
           - gitlab/envoy-filter.yaml
           - gitlab/gitaly-priority-class.yaml
     images:
-      - registry.gitlab.com/gitlab-org/build/cng/gitaly-init-cgroups:v17.2.9  # Remember to redeploy this package if changing stuff around this image in the gitlab package
+      - registry.gitlab.com/gitlab-org/build/cng/gitaly-init-cgroups:v17.6.1  # Remember to redeploy this package if changing stuff around this image in the gitlab package
 
   - name: pepr-policy-exemptions
     required: true

diff --git a/packages/object-store/gitlab/secret.yaml b/packages/object-store/gitlab/secret.yaml
@@ -16,7 +16,7 @@ stringData:
     path_style: true
   registry: |-
     s3:
-      bucket: uds-gitlab-registry
+      bucket: ###ZARF_VAR_REGISTRY_BUCKET###
       accesskey: ###ZARF_VAR_ACCESS_KEY###
       secretkey: ###ZARF_VAR_SECRET_KEY###
       regionendpoint: ###ZARF_VAR_ENDPOINT###

diff --git a/packages/object-store/gitlab/zarf.yaml b/packages/object-store/gitlab/zarf.yaml
@@ -12,6 +12,8 @@ variables:
     default: "replace-me"
   - name: SECRET_KEY
     default: "replace-me"
+  - name: REGISTRY_BUCKET
+    default: uds-gitlab-registry
 
 components:
   - name: gitlab-object-store-secret
Original file line number	Diff line number	Diff line change
Expand Up		@@ -20,4 +20,4 @@ You can confirm newly created repos are going to different shards by viewing the

		## Prevent Gitaly Shard OOM

		The UDS Config variable `GITALY_CGROUPS` allows adminstrators to improve the resiliency of each gitaly node. The proper values are heavily context specific so it is disabled by default. However, the value given to that variable will be substituted directly into the `gitlab.gitaly.cgroups` path [documented here](https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage). Adding these values will reduce the risk a single git call will cause the gitaly node (shard) to be killed with an OOM error.
		The UDS Config variable `GITALY_CGROUPS` allows administrators to improve the resiliency of each gitaly node. The proper values are heavily context specific so it is disabled by default. However, the value given to that variable will be substituted directly into the `gitlab.gitaly.cgroups` path [documented here](https://docs.gitlab.com/ee/administration/gitaly/kubernetes.html#constrain-git-processes-resource-usage). Adding these values will reduce the risk a single git call will cause the gitaly node (shard) to be killed with an OOM error.