diff --git a/.github/workflows/tag-and-release.yaml b/.github/workflows/tag-and-release.yaml index 5f0d3fd1..49c337dc 100644 --- a/.github/workflows/tag-and-release.yaml +++ b/.github/workflows/tag-and-release.yaml @@ -23,7 +23,7 @@ jobs: needs: tag-new-version if: ${{ needs.tag-new-version.outputs.release_created == 'true' }} # TODO: (@WSTARR) since we don't run the tests on arm currently we don't need to use the 8 core - runs-on: ${{ matrix.architecture == 'arm64' && 'uds-ubuntu-arm64-4-core' || 'uds-ubuntu-big-boy-8-core' }} + runs-on: ${{ matrix.architecture == 'arm64' && 'uds-swf-ubuntu-arm64-4-core' || 'uds-swf-ubuntu-big-boy-8-core' }} name: Publish ${{ matrix.flavor }} ${{ matrix.architecture }} timeout-minutes: 40 strategy: @@ -52,21 +52,21 @@ jobs: uses: defenseunicorns/uds-common/.github/actions/playwright@772b3337950b7c8e0882c527263684306bba7ce4 # v0.7.1 - name: Build Package - run: uds run -f tasks/publish.yaml build-package --set FLAVOR=${{ matrix.flavor }} + run: uds run -f tasks/publish.yaml build-package --set FLAVOR=${{ matrix.flavor }} --no-progress - name: Build Bundle - run: uds run -f tasks/publish.yaml build-test-bundle --set FLAVOR=${{ matrix.flavor }} + run: uds run -f tasks/publish.yaml build-test-bundle --set FLAVOR=${{ matrix.flavor }} --no-progress - name: Test Package if: ${{ runner.arch != 'ARM64' }} - run: uds run -f tasks/publish.yaml test-package --set FLAVOR=${{ matrix.flavor }} + run: uds run -f tasks/publish.yaml test-package --set FLAVOR=${{ matrix.flavor }} --no-progress - name: Publish Package - run: uds run -f tasks/publish.yaml publish-package --set FLAVOR=${{ matrix.flavor }} + run: uds run -f tasks/publish.yaml publish-package --set FLAVOR=${{ matrix.flavor }} --no-progress - name: Publish Bundle if: ${{ matrix.flavor == 'upstream' }} - run: uds run -f tasks/publish.yaml publish-test-bundle --set FLAVOR=${{ matrix.flavor }} + run: uds run -f tasks/publish.yaml publish-test-bundle --set FLAVOR=${{ matrix.flavor }} --no-progress - name: Debug Output if: ${{ always() }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 047241f3..5528dd7d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -34,7 +34,7 @@ permissions: jobs: run-test: name: ${{ matrix.type }} ${{ matrix.flavor }} - runs-on: uds-ubuntu-big-boy-8-core + runs-on: uds-swf-ubuntu-big-boy-8-core timeout-minutes: 25 strategy: matrix: diff --git a/bundle/uds-bundle.yaml b/bundle/uds-bundle.yaml index 4855470a..6daa8627 100644 --- a/bundle/uds-bundle.yaml +++ b/bundle/uds-bundle.yaml @@ -17,23 +17,56 @@ packages: - name: postgres-operator repository: ghcr.io/defenseunicorns/packages/uds/postgres-operator - ref: 1.10.1-uds.4-upstream + ref: 1.12.2-uds.1-upstream overrides: postgres-operator: uds-postgres-config: - variables: - - name: POSTGRESQL - description: "Configure postgres using CRs via the uds-postgres-config chart" - path: postgresql + values: + - path: postgresql + value: + enabled: true # Set to false to not create the PostgreSQL resource + teamId: "uds" + volume: + size: "10Gi" + numberOfInstances: 2 + users: + gitlab.gitlab: [] # database owner + databases: + gitlabdb: gitlab.gitlab + version: "14" + ingress: + - remoteNamespace: gitlab - - name: dev-redis - repository: ghcr.io/defenseunicorns/packages/uds/dev-redis - ref: 0.0.2 + - name: valkey + repository: ghcr.io/defenseunicorns/packages/uds/valkey + ref: 7.2.5-uds.1-upstream + overrides: + valkey: + uds-valkey-config: + values: + - path: custom + value: + - direction: Ingress + selector: + app.kubernetes.io/name: valkey + remoteNamespace: gitlab + port: 6379 + description: "Ingress from GitLab" + - path: copyPassword + value: + enabled: true + namespace: gitlab + secretName: gitlab-redis + secretKey: password - name: dev-secrets path: ../ ref: 0.1.0 + - name: dev-peer-auth-shim + path: ../ + ref: 0.1.0 + - name: gitlab path: ../ # x-release-please-start-version @@ -52,7 +85,23 @@ packages: - name: GITLAB_ADMIN_GROUPS description: "Array of group names that grant admin role gitlab when saml protocol is active." path: "sso.adminGroups" + values: + # TODO: (@WSTARR) The below two overrides will no longer be needed after the next release + - path: redis.namespace + value: valkey + - path: redis.selector + value: + app.kubernetes.io/name: valkey gitlab: + values: + - path: global.psql.host + value: pg-cluster.postgres.svc.cluster.local + - path: "global.psql.username" + value: "gitlab.gitlab" + - path: "global.psql.password.secret" + value: "gitlab.gitlab.pg-cluster.credentials.postgresql.acid.zalan.do" + - path: global.redis.host + value: valkey-master.valkey.svc.cluster.local variables: - name: GITLAB_SSO_ENABLED description: "Boolean to enable or disable sso things" @@ -87,11 +136,3 @@ packages: - name: SHELL_REPLICAS description: "Gitlab Shell Min Replicas" path: "gitlab.gitlab-shell.minReplicas" - - name: POSTGRES_USERNAME - description: "Gitlab Postgres Username" - path: "global.psql.username" - default: "gitlab.gitlab" - - name: POSTGRES_SECRET_REF - description: "Gitlab Postgres Password Secret Ref" - path: "global.psql.password.secret" - default: "gitlab.gitlab.pg-cluster.credentials.postgresql.acid.zalan.do" diff --git a/bundle/uds-config.yaml b/bundle/uds-config.yaml index 4e681b26..e3cd8e0f 100644 --- a/bundle/uds-config.yaml +++ b/bundle/uds-config.yaml @@ -13,23 +13,7 @@ variables: - name: uds-gitlab-uploads - name: uds-gitlab-registry - name: uds-gitlab-tmp - postgres-operator: - postgresql: - enabled: true # Set to false to not create the PostgreSQL resource - teamId: "uds" - volume: - size: "10Gi" - numberOfInstances: 2 - users: - gitlab.gitlab: [] # database owner - databases: - gitlabdb: gitlab.gitlab - version: "13" - ingress: - remoteGenerated: Anywhere gitlab: - gitlab_redis_endpoint: "redis-master.dev-redis.svc.cluster.local" - gitlab_db_endpoint: "pg-cluster.postgres.svc.cluster.local" DISABLE_REGISTRY_REDIRECT: "true" GITLAB_PAGES_ENABLED: true GITLAB_ADMIN_GROUPS: ["/GitLab Admin", "/UDS Core/Admin"] diff --git a/chart/templates/postgres-peerauthentication.yaml b/chart/templates/postgres-peerauthentication.yaml new file mode 100644 index 00000000..47c61c18 --- /dev/null +++ b/chart/templates/postgres-peerauthentication.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.postgres.internal (.Capabilities.APIVersions.Has "security.istio.io/v1beta1") }} +apiVersion: "security.istio.io/v1beta1" +kind: PeerAuthentication +metadata: + name: gitlab-postgres-peerauthentication-exception + namespace: {{ .Values.postgres.namespace }} +spec: + mtls: + mode: STRICT + selector: + matchLabels: + {{ .Values.postgres.selector | toYaml }} + portLevelMtls: + # Postgres exception to support GitLab dependency init containers + {{ .Values.postgres.port }}: + mode: PERMISSIVE +{{- end }} diff --git a/chart/templates/redis-peerauthentication.yaml b/chart/templates/redis-peerauthentication.yaml new file mode 100644 index 00000000..1cf18684 --- /dev/null +++ b/chart/templates/redis-peerauthentication.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.redis.internal (.Capabilities.APIVersions.Has "security.istio.io/v1beta1") }} +apiVersion: "security.istio.io/v1beta1" +kind: PeerAuthentication +metadata: + name: gitlab-redis-peerauthentication-exception + namespace: {{ .Values.redis.namespace }} +spec: + mtls: + mode: STRICT + selector: + matchLabels: + {{ .Values.redis.selector | toYaml }} + portLevelMtls: + # Redis/Valkey exception to support GitLab dependency init containers + {{ .Values.redis.port }}: + mode: PERMISSIVE +{{- end }} diff --git a/chart/templates/redis-secret.yaml b/chart/templates/redis-secret.yaml new file mode 100644 index 00000000..c41628ad --- /dev/null +++ b/chart/templates/redis-secret.yaml @@ -0,0 +1,10 @@ +{{- if ne .Values.redis.password "" }} +apiVersion: v1 +kind: Secret +metadata: + name: gitlab-redis + namespace: {{ .Release.Namespace }} +type: kubernetes.io/opaque +stringData: + password: {{ .Values.redis.password }} +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index f1cb57eb..cb09bf3d 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -12,11 +12,13 @@ storage: namespace: dev-minio port: 9000 redis: + password: "" + # Set to false to use external redis internal: true selector: - app.kubernetes.io/name: redis - namespace: dev-redis + app.kubernetes.io/name: valkey + namespace: valkey port: 6379 postgres: password: "" diff --git a/docs/configuration.md b/docs/configuration.md index 329bcd96..547423d1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,12 +17,12 @@ GitLab uses Postgres as its backing database service and supports the [common da ### Manual Database Connection -If you are using the UDS Postgres Operator or another external database that uses usernames/passwords you can use the following Helm overrides to configure it: +If you are using the [UDS Postgres Operator](https://github.com/defenseunicorns/uds-package-postgres-operator/) or another external database that uses usernames/passwords you can use the following Helm overrides to configure it: #### `uds-gitlab-config` chart: > [!IMPORTANT] -> The `postgres.password` setting is not applicable when using the UDS Postgres Operator package or when supplying a secret manually. +> The `postgres.password` setting is not applicable when using the UDS Postgres Operator package or when supplying a secret manually! - `postgres.password` - provides a password to generate a secret to pass to GitLab @@ -30,14 +30,42 @@ If you are using the UDS Postgres Operator or another external database that use #### `gitlab` chart: > [!IMPORTANT] -> The `global.psql.password.secret` setting is not applicable when providing a password to the `uds-gitlab-config` chart manually. +> The `global.psql.password.secret` and `global.psql.password.key` settings are not applicable when providing a password to the `uds-gitlab-config` chart manually. - `global.psql.username` - provides the username to use when connecting to the database (i.e. `gitlab.gitlab`) -- `global.psql.password.secret` - provides the secret that contains the database password (i.e. `gitlab.gitlab.pg-cluster.credentials.postgresql.acid.zalan.do`) +- `global.psql.password.secret` - provides the secret that contains the database password (defaults to `gitlab-postgres`) +- `global.psql.password.key` - provides the secret key that contains the database password (defaults to `password`) - `global.psql.host` - provides the endpoint to use to connect to the database (i.e. `pg-cluster.postgres.svc.cluster.local`) +- `global.psql.port` - provides the port to use to connect to the database (defaults to `5432`) ### IAM Roles for Service Accounts The Software Factory team has not yet tested IRSA with AWS RDS - there is an open issue linked below with further linked issues to test this that could act as a starting point to implement: https://github.com/defenseunicorns/uds-software-factory/issues/45 + +## Redis / Valkey + +GitLab uses Redis as a key value store for caching, job queueing and more and supports external providers (such as Elasticache) as well as the [UDS Valkey](https://github.com/defenseunicorns/uds-package-valkey/) package to provide the service. + +### Manual Database Connection + +You can use the following Helm overrides to configure a connection to Redis / Valkey: + +#### `uds-gitlab-config` chart: + +> [!IMPORTANT] +> The `redis.password` setting is not applicable when using the UDS Valkey package or when supplying a secret manually! + +- `redis.password` - provides a password to generate a secret to pass to GitLab + +#### `gitlab` chart: + +> [!IMPORTANT] +> The `global.redis.auth.secret` and `global.redis.auth.key` settings are not applicable when providing a password to the `uds-gitlab-config` chart manually. + +- `global.redis.auth.secret` - provides the secret that contains the key value store password (defaults to `gitlab-redis`) +- `global.redis.auth.key` - provides the key within the secret that contains the key value store password (defaults to `password`) +- `global.redis.scheme` - provides the scheme to use to connect to the key value store (i.e. `redis` or `rediss`) +- `global.redis.host` - provides the endpoint to use to connect to the key value store (i.e. `pg-cluster.postgres.svc.cluster.local`) +- `global.redis.port` - provides the port to use to connect to the key value store (defaults to `6379`) diff --git a/src/dev-secrets/redis-secret.yaml b/src/dev-secrets/redis-secret.yaml deleted file mode 100644 index 551c4aaf..00000000 --- a/src/dev-secrets/redis-secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: gitlab-redis - namespace: gitlab -type: kubernetes.io/opaque -stringData: - password: "###ZARF_VAR_REDIS_PASSWORD###" diff --git a/src/dev-secrets/zarf.yaml b/src/dev-secrets/zarf.yaml index 2333ffbf..c8d97109 100644 --- a/src/dev-secrets/zarf.yaml +++ b/src/dev-secrets/zarf.yaml @@ -27,20 +27,3 @@ components: - name: gitlab-minio files: - "minio-secret.yaml" - - name: redis-password - required: true - actions: - onDeploy: - before: - - cmd: ./zarf tools kubectl get secret -n dev-redis redis --template='{{ index .data "redis-password" }}' | base64 -d - mute: true - setVariables: - - name: REDIS_PASSWORD - sensitive: true - - name: gitlab-redis - required: true - manifests: - - name: gitlab-redis - namespace: gitlab - files: - - redis-secret.yaml diff --git a/src/peer-auth-shim/postgres-peerauthentication.yaml b/src/peer-auth-shim/postgres-peerauthentication.yaml new file mode 100644 index 00000000..e63a8b5e --- /dev/null +++ b/src/peer-auth-shim/postgres-peerauthentication.yaml @@ -0,0 +1,15 @@ +apiVersion: "security.istio.io/v1beta1" +kind: PeerAuthentication +metadata: + name: gitlab-postgres-peerauthentication-shim-exception + namespace: postgres +spec: + mtls: + mode: STRICT + selector: + matchLabels: + cluster-name: pg-cluster + portLevelMtls: + # Postgres exception to support GitLab dependency init containers + 5432: + mode: PERMISSIVE diff --git a/src/peer-auth-shim/redis-peerauthentication.yaml b/src/peer-auth-shim/redis-peerauthentication.yaml new file mode 100644 index 00000000..754495be --- /dev/null +++ b/src/peer-auth-shim/redis-peerauthentication.yaml @@ -0,0 +1,15 @@ +apiVersion: "security.istio.io/v1beta1" +kind: PeerAuthentication +metadata: + name: gitlab-redis-peerauthentication-shim-exception + namespace: valkey +spec: + mtls: + mode: STRICT + selector: + matchLabels: + app.kubernetes.io/name: valkey + portLevelMtls: + # Redis/Valkey exception to support GitLab dependency init containers + 6379: + mode: PERMISSIVE diff --git a/src/peer-auth-shim/zarf.yaml b/src/peer-auth-shim/zarf.yaml new file mode 100644 index 00000000..ebc845a3 --- /dev/null +++ b/src/peer-auth-shim/zarf.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json +# TODO: (@WSTARR) This will no longer be needed after the next release of GitLab +kind: ZarfPackageConfig +metadata: + name: dev-peer-auth-shim + version: "0.1.0" + +components: + - name: gitlab-peer-auth-shim + required: true + manifests: + - name: gitlab-peer-auth-shim + files: + - "postgres-peerauthentication.yaml" + - "redis-peerauthentication.yaml" diff --git a/tasks/dependencies.yaml b/tasks/dependencies.yaml index 8c8d8fbf..e74987c5 100644 --- a/tasks/dependencies.yaml +++ b/tasks/dependencies.yaml @@ -10,3 +10,4 @@ tasks: actions: - cmd: ./uds zarf package create src/dev-secrets/ --confirm --no-progress --architecture=${{ .inputs.architecture }} --skip-sbom ${{ .inputs.options }} - cmd: ./uds zarf package create src/namespace/ --confirm --no-progress --architecture=${{ .inputs.architecture }} --skip-sbom ${{ .inputs.options }} + - cmd: ./uds zarf package create src/peer-auth-shim/ --confirm --no-progress --architecture=${{ .inputs.architecture }} --skip-sbom ${{ .inputs.options }} diff --git a/zarf.yaml b/zarf.yaml index b18e111c..6c3fa9d8 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -15,7 +15,7 @@ variables: - name: BUCKET_SUFFIX default: "" - name: GITLAB_REDIS_ENDPOINT - default: "redis-master" + default: "" - name: GITLAB_REDIS_SCHEME default: "redis" - name: GITLAB_DB_NAME