feat: add pgbouncer configuration for gitlab

bundles/rke2/uds-bundle.yaml

@@ -418,17 +418,111 @@ packages:
                 secretName: gitlab-redis
                 secretKey: password
 
+  - name: gitlab-pgbouncer-rw
+    repository: ghcr.io/defenseunicorns/packages/uds/pgbouncer
+    ref: 1.21.0-uds.5-registry1
+    exports:
+      - name: GITLAB_DB_USERNAME
+      - name: GITLAB_DB_PASSWORD
+    overrides:
+      pgbouncer:
+        pgbouncer:
+          namespace: gitlab-pgbouncer-rw
+          variables:
+            - name: GITLAB_DB_USERNAME
+              path: config.adminUser
+              description: "pgbouncer admin username. Should match Gitlab postgres database username"
+            - name: GITLAB_DB_PASSWORD
+              path: config.adminPassword
+              description: "pgbouncer admin password. Should match GitLab postgres database password"
+            - name: PGBOUNCER_DATABASES
+              path: config.databases
+              description: "Database configuration for pgbouncer"
+            - name: PGBOUNCER_REPLICA_COUNT
+              path: replicaCount
+              description: "Number of desired PgBouncer pods"
+              default: 3
+            - name: PGBOUNCER_CONFIG
+              path: config.pgbouncer
+              description: "Optional variable that should only need configured if the default pgbouncer configuration is inadequate."
+        uds-pgbouncer-config:
+          namespace: gitlab-pgbouncer-rw
+          values:
+            - path: pgbouncer.ingress
+              value:
+                - remoteNamespace: gitlab
+            - path: pgbouncer.peerauth
+              value:
+                permissive: true
+                port: 5432
+            - path: postgres.internal
+              value: false
+              description: "Postgres DB is external to cluster, so set this to false to all egress traffic out of cluster"
+
+  # Temporary read replica pgbouncer cluster. Not ideal setup for gitlab's internal read replica loadbalancing logic.
+  # This will configure multiple pgbouncer instances to connect to the read replicas via HA proxy.
+  # GitLab recommends a single read replica pgbouncer per postgres replica and to configure gitlab to load balance reads to each replica directly.
+  # https://docs.gitlab.com/ee/administration/postgresql/database_load_balancing.html
+  - name: gitlab-pgbouncer-ro
+    repository: ghcr.io/defenseunicorns/packages/uds/pgbouncer
+    ref: 1.21.0-uds.5-registry1
+    imports:
+      - name: GITLAB_DB_USERNAME
+        package: gitlab-pgbouncer-rw
+      - name: GITLAB_DB_PASSWORD
+        package: gitlab-pgbouncer-rw
+    overrides:
+      pgbouncer:
+        pgbouncer:
+          namespace: gitlab-pgbouncer-ro
+          variables:
+            - name: GITLAB_DB_USERNAME
+              path: config.adminUser
+              description: "pgbouncer admin username. Should match Gitlab postgres database username"
+            - name: GITLAB_DB_PASSWORD
+              path: config.adminPassword
+              description: "pgbouncer admin password. Should match GitLab postgres database password"
+            - name: PGBOUNCER_DATABASES
+              path: config.databases
+              description: "Database configuration for pgbouncer"
+            - name: PGBOUNCER_REPLICA_COUNT
+              path: replicaCount
+              description: "Number of desired PgBouncer pods"
+              default: 3
+            - name: PGBOUNCER_CONFIG
+              path: config.pgbouncer
+              description: "Optional variable that should only need configured if the default pgbouncer configuration is inadequate."
+        uds-pgbouncer-config:
+          namespace: gitlab-pgbouncer-ro
+          values:
+            - path: pgbouncer.ingress
+              value:
+                - remoteNamespace: gitlab
+            - path: pgbouncer.peerauth
+              value:
+                permissive: true
+                port: 5432
+            - path: postgres.internal
+              value: false
+              description: "Postgres DB is external to cluster, so set this to false to all egress traffic out of cluster"
+
   - name: gitlab-object-store
     path: ../../build
     ref: 0.0.1
 
   - name: gitlab-database-secret
     path: ../../build
     ref: 0.0.1
+    imports:
+      - name: GITLAB_DB_PASSWORD
+        package: gitlab-pgbouncer-rw
 
   - name: gitlab
     repository: ghcr.io/defenseunicorns/packages/uds/gitlab
     ref: 17.2.9-uds.0-registry1
+    imports:
+      - name: GITLAB_DB_USERNAME
+        package: gitlab-pgbouncer-rw
     overrides:
       gitlab:
         uds-gitlab-config:
@@ -490,6 +584,9 @@ packages:
                 runAsNonRoot: true
                 capabilities:
                   drop: ["ALL"]
+            - path: global.psql.host
+              description: "GitLab DB host should point to kubernetes service for read/write pgbouncer cluster."
+              value: "pgbouncer.gitlab-pgbouncer-rw.svc.cluster.local"
           variables:
             - name: MIGRATIONS_RESOURCES
               description: "Gitlab Migrations Resources"
@@ -567,6 +664,29 @@ packages:
               path: global.gitaly.internal.names # Gitlab uses the global top-value to set subcharts: https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/values.yaml?ref_type=heads#L184
               default:
                 - "default"  # Gitlab requires that the first shard always be named `default`: https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html?tab=Helm+chart+%28Kubernetes%29#gitlab-requires-a-default-repository-storage
+            # https://docs.gitlab.com/charts/charts/globals.html#postgresql-load-balancing - configure read replica load balancing hosts list and fine tuning if needed.
+            - name: GITLAB_DB_LOAD_BALANCING
+              path: global.psql.load_balancing
+              description: "GitLab read replica load balancing configuration"
+              default: # Since we are currently using in cluster pgbouncer for read replicas, set default to only use hostname for kubernetes service for read pgbouncer cluster
+                hosts:
+                  - "pgbouncer.gitlab-pgbouncer-ro.svc.cluster.local"
+            - name: MIGRATIONS_DB_ENDPOINT
+              description: "Postgres hostname/IP for gitlab migrations job to run against. Must bypass pgbouncer for migrations so this must be either HA proxy or direct connection to postgres primary."
+              path: "gitlab.migrations.psql.host"
+            - name: MIGRATIONS_DB_PORT
+              description: "Postgres port for gitlab migrations job."
+              path: "gitlab.migrations.psql.port"
+            - name: TOOLBOX_DB_ENDPOINT
+              description: "Postgres hostname/IP for gitlab toolbox. Must bypass pgbouncer for backups so this must be either HA proxy or direct connection to postgres primary."
+              path: "gitlab.toolbox.psql.host"
+            - name: TOOLBOX_DB_PORT
+              description: "Postgres port for gitlab toolbox."
+              path: "gitlab.toolbox.psql.port"
+            - name: GITLAB_DB_USERNAME
+              description: "postgres username"
+              path: "global.psql.username"
+              default: "postgres"
 
   # Gitlab Runner
   - name: gitlab-runner

config/uds-config.yaml

@@ -89,16 +89,28 @@ variables:
     VELERO_BUCKET_REGION: "us-east-1"
     VELERO_BUCKET_KEY: "replace-me-object-store-access-key"
     VELERO_BUCKET_KEY_SECRET: "replace-me-object-store-secret-key"
-  gitlab-database-secret:
-    GITLAB_DB_PASSWORD: "replace-me-db-passwords"
+  gitlab-pgbouncer-rw:
+    GITLAB_DB_PASSWORD: ""
+    GITLAB_DB_USERNAME: postgres
+    PGBOUNCER_DATABASES:
+      gitlabdb:
+        host: "gitlab-pg.replace.with.db.url"
+        port: 5000 # Current write port configured for HA proxy by ndb-pg-db-da IaC module
+  gitlab-pgbouncer-ro:
+    PGBOUNCER_DATABASES:
+      gitlabdb:
+        host: "gitlab-pg.replace.with.db.url"
+        port: 5001 # Current read port configured for HA proxy by ndb-pg-db-da IaC module
   gitlab-object-store:
     ENDPOINT: "http://replace.with.object.store.url"
     ACCESS_KEY: "replace-me-object-store-access-key"
     SECRET_KEY: "replace-me-object-store-secret-key"
   gitlab:
     GITLAB_DB_NAME: "gitlabdb"
-    GITLAB_DB_USERNAME: "postgres"
-    GITLAB_DB_ENDPOINT: "gitlab-pg.replace.with.db.url"
+    MIGRATIONS_DB_ENDPOINT: "gitlab-pg.replace.with.db.url"
+    TOOLBOX_DB_ENDPOINT: "gitlab-pg.replace.with.db.url"
+    MIGRATIONS_DB_PORT: 5000
+    TOOLBOX_DB_PORT: 5000
     GITLAB_PAGES_ENABLED: "true"
     GITLAB_SIGNUP_ENABLED: "false"
     GITLAB_BACKUP_SCHEDULE: "0 3 * * *"
@@ -159,6 +171,14 @@ variables:
       cpuQuotaUs: 1400000 # 14 cores
     REGISTRY_REPLICAS: 2
     SHELL_REPLICAS: 2
+    GITLAB_DB_LOAD_BALANCING:
+      hosts:
+        - replica-0.database
+        - replica-1.database
+        - replica-2.database
+      max_replication_difference: # Optional. See documentation https://docs.gitlab.com/ee/administration/postgresql/database_load_balancing.html#handling-stale-reads
+      max_replication_lag_time:   # Optional. See documentation
+      replica_check_interval:     # Optional. See documentation
   sonarqube:
     SONARQUBE_DB_NAME: "sonarqubedb"
     SONARQUBE_DB_USERNAME: "postgres"