feat: add SETUID and SETGID capabilities for gitlab runner container …

…security context (#116)

## Description

Add SETUID and SETGID capability for gitlab runner container security context to
allow running buildah and podman inside a container from the runners.

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [x] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [ ] [Contributor Guide
Steps](https://github.com/defenseunicorns/uds-package-gitlab-runner/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Co-authored-by: Wayne Starr <Racer159@users.noreply.github.com>

chart/templates/uds-policy-exemptions.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.enableSecurityCapabilities }}
+apiVersion: uds.dev/v1alpha1
+kind: Exemption
+metadata:
+  name: gitlab-runner-container-building
+  namespace: uds-policy-exemptions
+spec:
+  exemptions:
+    - description: Allow more capabilities for container build tools (Buildah) to be able to map user and group IDs
+      policies:
+        - RestrictCapabilities
+      title: "gitlab-runner-container-building"
+      matcher:
+        namespace: gitlab-runner-sandbox
+        name: "^runner-.*"
+{{- end }}

chart/values.yaml

@@ -7,6 +7,8 @@ serviceAccountName: "gitlab-runner"
 
 runnerAuthToken: "###ZARF_VAR_RUNNER_AUTH_TOKEN###"
 
+enableSecurityCapabilities: false
+
 custom: []
   # - direction: Egress
   #   remoteGenerated: Anywhere

common/zarf.yaml

@@ -12,6 +12,8 @@ components:
         namespace: gitlab-runner
         version: 0.1.0
         localPath: ../chart
+        valuesFiles:
+          - ../values/config-values.yaml
       - name: gitlab-runner
         namespace: gitlab-runner
         url: https://charts.gitlab.io

docs/configuration.md

@@ -2,6 +2,18 @@
 
 GitLab Runners in this package are configured through the upstream [GitLab Runner chart](https://docs.gitlab.com/runner/install/kubernetes.html) as well as a UDS configuration chart that supports the following:
 
+## Node Configuration
+
+> [!IMPORTANT]
+> Any kubernetes node that will run GitLab Runner pods to use tooling like [Buildah](https://buildah.io/) must set sysctl `user.max_user_namespaces` to a nonzero value. This is required to run these container builds inside Linux containers from the runner pods.
+>
+> This is a [STIG finding](https://www.stigviewer.com/stig/red_hat_enterprise_linux_9/2023-09-13/finding/V-257816) but is `Not Applicable` when running Linux containers.
+
+Example:
+```bash
+sysctl -w user.max_user_namespaces=30110
+```
+
 ## Networking
 
 Network policies are controlled via the `uds-gitlab-runner-config` chart in accordance with the [common patterns for networking within UDS Software Factory](https://github.com/defenseunicorns/uds-software-factory/blob/main/docs/networking.md).  Because GitLab runners do not interact with external resources like databases or object storage they only implement `custom` networking for both the runner namespace and the runner sandbox namespace:
@@ -37,6 +49,12 @@ By default the sandbox is excluded from being mutated by Zarf to allow external
 > [!TIP]
 > The default registry behavior relies on the `###ZARF_REGISTRY###` internal value as outlined in the [Zarf documentation](https://docs.zarf.dev/ref/values/#internal-values-zarf).  This value is applied during Zarf deploy so cannot be used by GitLab when spawning pods.  If you do know the address of the Zarf registry (`127.0.0.1:31999` by default) you can still pull from the Zarf registry however.
 
+### Allow SETUID and SETGID security capabilities
+
+By default, runner build containers do not have `SETUID` and `SETGID` capabilities enabled. This limits the functionality of tools like [Buildah](https://buildah.io/) and [Podman](https://podman.io/). Podman cannot build container images, and Buildah can only create very basic images. Any actions that involve user or group modifications (e.g., using useradd or groupadd in a Dockerfile) will fail.
+
+To enable `SETUID` and `SETGID` capabilities in the build containers, set the `ENABLE_SECURITY_CAPABILITIES` Zarf variable to `true`. This will [apply a security policy for the build container](https://docs.gitlab.com/runner/executors/kubernetes/#set-a-security-policy-for-the-container) to add SETUID and SETGID capabilities. Additionally, it will [add a UDS Policy Exemption](https://uds.defenseunicorns.com/core/configuration/uds-configure-policy-exemptions/) to permit these capabilities.
+
 ### Change the Runner Service Account
 
 By default the chart will create a service account named `gitlab-runner`.  You can change the name of this service account by by overriding the `serviceAccountName` value in the `uds-gitlab-runner-config` chart along with the `rbac.generatedServiceAccountName` value in the `gitlab-runner` chart.

tasks/test.yaml

@@ -8,14 +8,13 @@ tasks:
       # Ensure all GL services are up
       - task: gitlab-ingress
       # Run checks on initial deployment
-      - task: glr-health-check
-      - task: glr-run-check
+      - task: glr-registration-check
+      - task: glr-run-check-default-security-capabilities
       # Create a runner token and hide the secret from the GLR package
       - task: glr-create-runner-token
       - task: glr-backup-registration-secret
       # Remove the GLR package and redeploy with the manual token
       - task: remove:test-bundle
-      # TODO: (@WSTARR) Maru will complain about "cyclical" task imports if this imports the deploy task from uds-common.  This is a bug: https://github.com/defenseunicorns/maru-runner/issues/122
       - description: Get the current UDS Bundle name
         cmd: cat bundle/uds-bundle.yaml | ./uds zarf tools yq .metadata.name
         setVariables:
@@ -24,11 +23,13 @@ tasks:
         cmd: cat bundle/uds-bundle.yaml | ./uds zarf tools yq .metadata.version
         setVariables:
           - name: BUNDLE_VERSION
+      # TODO: (@WSTARR) Maru will complain about "cyclical" task imports if this imports the deploy task from uds-common.  This is a bug: https://github.com/defenseunicorns/maru-runner/issues/122
       - description: Deploys the current GitLab runner package
-        cmd: UDS_CONFIG=bundle/uds-config.yaml ./uds deploy bundle/uds-bundle-${BUNDLE_NAME}-${UDS_ARCH}-${BUNDLE_VERSION}.tar.zst --confirm --no-progress --set RUNNER_AUTH_TOKEN=${RUNNER_AUTH_TOKEN}
+        cmd: UDS_CONFIG=bundle/uds-config.yaml ./uds deploy bundle/uds-bundle-${BUNDLE_NAME}-${UDS_ARCH}-${BUNDLE_VERSION}.tar.zst --confirm --no-progress --set RUNNER_AUTH_TOKEN=${RUNNER_AUTH_TOKEN} --set ENABLE_SECURITY_CAPABILITIES=true
       # Check that the runner registered and restore the secret
-      - task: glr-health-check
+      - task: glr-registration-check
       - task: glr-restore-registration-secret
+      - task: glr-run-check-elevated-security-capabilities
 
   - name: gitlab-ingress
     actions:
@@ -45,7 +46,7 @@ tasks:
             exit 1
           fi
 
-  - name: glr-health-check
+  - name: glr-registration-check
     description: Check the status of Gitlab Runner
     actions:
       - description: Check Gitlab Runner Secret
@@ -58,12 +59,24 @@ tasks:
         dir: tests
         cmd: npm test -- journey/registration.test.ts
 
-  - name: glr-run-check
+  - name: glr-run-check-default-security-capabilities
+    description: Check that a GitLab repository can trigger a gitlab runner to run
+    actions:
+      - description: Setup a repository and trigger a pipeline job
+        dir: tests
+        cmd: |
+          npm test -- journey/pipeline-run.test.ts -t 'hello kitteh succeeds'
+          npm test -- journey/pipeline-run.test.ts -t 'podman fails'
+
+  - name: glr-run-check-elevated-security-capabilities
     description: Check that a GitLab repository can trigger a gitlab runner to run
     actions:
       - description: Setup a repository and trigger a pipeline job
         dir: tests
-        cmd: npm test -- journey/pipeline-run.test.ts
+        cmd: |
+          npm test -- journey/pipeline-run.test.ts -t 'hello kitteh succeeds'
+          npm test -- journey/pipeline-run.test.ts -t 'podman succeeds'
+
 
   - name: glr-create-runner-token
     description: Create a runner auth token and set the variable RUNNER_AUTH_TOKEN

tests/journey/pipeline-run.test.ts

@@ -1,10 +1,61 @@
 import { expect, test} from '@jest/globals';
 import { K8s, kind } from "kubernetes-fluent-client";
 import { zarfExec, retry } from "../common";
+import * as path from 'path';
+import { execSync } from 'child_process';
+import { rm } from 'fs/promises';
+
+const domainSuffix = process.env.DOMAIN_SUFFIX || ".uds.dev"
+
+test('hello kitteh succeeds', async () => {
+    const sourceRepoName = 'kitteh'
+    const expectedStatus = 'success'
+    const expectedJobLogOutputs: string[] = ['Hello Kitteh']
+
+    await executeTest(sourceRepoName, expectedJobLogOutputs, expectedStatus)
+}, 90000);
+
+
+test('podman succeeds', async () => {
+    const sourceRepoName = 'podman'
+    const expectedStatus = 'success'
+    const expectedJobLogOutputs: string[] = ['STEP 1/2: FROM scratch', 'STEP 2/2: ADD test.txt /', 'COMMIT']
+
+    await executeTest(sourceRepoName, expectedJobLogOutputs, expectedStatus)
+}, 90000);
+
+
+test('podman fails', async () => {
+
+    const sourceRepoName = 'podman'
+    const expectedStatus = 'failed'
+    const expectedJobLogOutputs: string[] = []
+
+    await executeTest(sourceRepoName, expectedJobLogOutputs, expectedStatus)
+}, 90000);
+
+
+async function executeTest(sourceRepoName: string, expectedJobLogOutputs: string[], expectedStatus: string) {
+    const nowMillis = Date.now()
+    const tokenName = `if-you-see-me-in-production-something-is-horribly-wrong-${nowMillis}`    
+
+    var sourceDir = path.join(__dirname, 'repo-sources', sourceRepoName)
 
-test('test kicking off a pipeline run', async () => {    
     // Get the toolbox pod and add a token to the root GitLab user
-    const tokenName = `if-you-see-me-in-production-something-is-horribly-wrong-${new Date()}`
+    await createToken(tokenName, nowMillis)
+    const headers: HeadersInit = [["PRIVATE-TOKEN", tokenName]]
+
+    const gitLabProjectName = `${sourceRepoName}-${nowMillis}`
+    const projectId = await createNewGitlabProject(sourceDir, tokenName, gitLabProjectName, headers)
+
+    await unprotectRunner(headers, tokenName)
+
+    // Check that the pipeline ran as expected
+    await checkJobResults(projectId, headers, expectedJobLogOutputs, expectedStatus)
+}
+
+
+async function createToken(tokenName: string, nowMillis: number) {
     const toolboxPods = await K8s(kind.Pod).InNamespace("gitlab").WithLabel("app", "toolbox").Get()
     const toolboxPod = toolboxPods.items.at(0)
     zarfExec(["tools",
@@ -14,57 +65,73 @@ test('test kicking off a pipeline run', async () => {
         "-i",
         toolboxPod?.metadata?.name!,
         "--",
-        `gitlab-rails runner "token = User.find_by_username('root').personal_access_tokens.create(scopes: ['api', 'admin_mode', 'read_repository', 'write_repository'], name: 'Root Test Token', expires_at: 1.days.from_now); token.set_token('${tokenName}'); token.save!"`
-     ]);
-
-    const arch = process.env.UDS_ARCH
-    // Create a test repository in GitLab using Zarf
-    zarfExec(["package", "create", "package", "--confirm"]);
-    zarfExec([
-        "package",
-        "mirror-resources",
-        `zarf-package-gitlab-runner-test-${arch}-0.0.1.tar.zst`,
-        "--git-url", "https://gitlab.uds.dev/",
-        "--git-push-username", "root",
-        "--git-push-password", `"${tokenName}"`,
-        "--confirm"
+        `gitlab-rails runner "token = User.find_by_username('root').personal_access_tokens.create(scopes: ['api', 'admin_mode', 'read_repository', 'write_repository'], name: 'Root Test Token ${nowMillis}', expires_at: 1.days.from_now); token.set_token('${tokenName}'); token.save!"`
     ]);
-
-    const headers: HeadersInit = [["PRIVATE-TOKEN", tokenName]]
+}
+
+async function createNewGitlabProject(sourceDir: string, tokenName: string, gitLabProjectName: string, headers: HeadersInit) {
+    await deleteDirectory(path.join(sourceDir, '.git')) 
+    execSync('git init', { cwd: sourceDir })
+    execSync('git add . ', { cwd: sourceDir })
+    execSync('git config commit.gpgsign false', { cwd: sourceDir }) // need this so that gpg signing doesn't attempt to happen locally when running tests
+    execSync('git commit -m "Initial commit" ', { cwd: sourceDir })
+    execSync(`git remote add origin https://root:${tokenName}@gitlab${domainSuffix}/root/${gitLabProjectName}.git`, { cwd: sourceDir })
+    execSync('git push -u origin --all', { cwd: sourceDir })
+    await deleteDirectory(path.join(sourceDir, '.git'))
 
-    // Un-protect the runner so that it picks up jobs from the `zarf-` branches
-    const runnerIDResp = await (await fetch(`https://gitlab.uds.dev/api/v4/runners/all`, { headers  })).json()
+    console.log(`Finding project id for project name [${encodeURIComponent(gitLabProjectName)}]`)
+    const projectResp = await fetch(`https://gitlab${domainSuffix}/api/v4/projects?search=${encodeURIComponent(gitLabProjectName)}`, { headers })
+    const projects = await projectResp.json()
+
+    const project = projects.find((p: { name: string; }) => p.name === gitLabProjectName)
+    const projectId = project?.id
+    console.log(`Found project id [${projectId}]`)
+    return projectId
+}
+
+async function unprotectRunner(headers: HeadersInit, tokenName: string) {
+    const runnerIDResp = await (await fetch(`https://gitlab${domainSuffix}/api/v4/runners/all`, { headers })).json()
     const runnerID = runnerIDResp[0].id
-    const runnerResp = await fetch(`https://gitlab.uds.dev/api/v4/runners/${runnerID}`, {
+    const runnerResp = await fetch(`https://gitlab${domainSuffix}/api/v4/runners/${runnerID}`, {
         headers: [
             ["PRIVATE-TOKEN", tokenName],
             ["Content-Type", "application/x-www-form-urlencoded"]
         ],
         body: "access_level=not_protected",
         method: "put"
-    })
+    });
     expect(runnerResp.status).toBe(200)
+}
 
-    // Check that the pipeline actually ran successfully
-    let foundTheKitteh = await retry(async () => {
-        const jobIDResp = await (await fetch(`https://gitlab.uds.dev/api/v4/projects/1/jobs`, { headers })).json()
+async function checkJobResults(projectId: any, headers: HeadersInit, expectedJobLogOutputs: string[], expectedStatus: string) {
+    let status = await retry(async () => {
+        const jobIDResp = await (await fetch(`https://gitlab${domainSuffix}/api/v4/projects/${projectId}/jobs`, { headers })).json()
 
         // Print the job response (useful for debugging)
         console.log(jobIDResp)
 
-        if (jobIDResp.length > 0 && jobIDResp[0].status === "success") {
-            const jobID = jobIDResp[0].id
-            const jobLog = await (await fetch(`https://gitlab.uds.dev/api/v4/projects/1/jobs/${jobID}/trace`, { headers })).text()
+        if (jobIDResp.length > 0 && (jobIDResp[0].status === "success" || jobIDResp[0].status === "failed")) {
+            const jobID = jobIDResp[0].id;
+            const jobLog = await (await fetch(`https://gitlab${domainSuffix}/api/v4/projects/${projectId}/jobs/${jobID}/trace`, { headers })).text()
 
             // Print the job log (useful for debugging)
             console.log(jobLog)
 
-            if (jobLog.indexOf("Hello Kitteh") > -1) {
-                return true
-            }
+            expectedJobLogOutputs.forEach( expectedOutput => {
+                expect(jobLog).toContain(expectedOutput)
+            });
+            return jobIDResp[0].status
         }
         return false
     }, 7, 7000);
-    expect(foundTheKitteh).toBe(true)
+    expect(status).toBe(expectedStatus)
+}
 
-}, 90000);
+async function deleteDirectory(path: string) {
+    try {
+        await rm(path, { recursive: true, force: true })
+        console.log(`Directory ${path} has been deleted successfully.`)
+    } catch (error) {
+        console.error(`Error while deleting directory ${path}:`, error)
+    }
+}

tests/journey/repo-sources/podman/.gitlab-ci.yml

@@ -0,0 +1,5 @@
+build-image-podman:
+  stage: build
+  image: quay.io/podman/stable:latest
+  script: |
+    podman build .

tests/journey/repo-sources/podman/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+ADD test.txt /

tests/journey/repo-sources/podman/test.txt

@@ -0,0 +1 @@
+text file that will get added to the container, referenced in Dockerfile

values/common-values.yaml

@@ -10,6 +10,8 @@ rbac:
       resources: [""]
       verbs: [""]
 
+enableSecurityCapabilities: ###ZARF_VAR_ENABLE_SECURITY_CAPABILITIES###
+
 runners:
   secret: gitlab-gitlab-runner-secret
   runUntagged: true
@@ -33,6 +35,11 @@ runners:
         "uds/user" = "${UDS_RUN_AS_USER}"
         "uds/group" = "${UDS_RUN_AS_GROUP}"
         "uds/network-access-gitlab" = "true"
+      {{- if .Values.enableSecurityCapabilities }}
+      [runners.kubernetes.build_container_security_context]
+        [runners.kubernetes.build_container_security_context.capabilities]
+          add = ["SETUID", "SETGID"]
+      {{- end }}
       [runners.kubernetes.helper_container_security_context]
         run_as_non_root = true
         run_as_user = 1001

values/config-values.yaml

@@ -0,0 +1 @@
+enableSecurityCapabilities: ###ZARF_VAR_ENABLE_SECURITY_CAPABILITIES###

zarf.yaml

@@ -14,6 +14,8 @@ variables:
     description: The Runner Authentication Token to use when registering the GitLab Runner (if none is provided will register a default instance runner)
   - name: RUNNER_SANDBOX_NAMESPACE
     default: gitlab-runner-sandbox
+  - name: ENABLE_SECURITY_CAPABILITIES
+    default: "false"
 
 components:
   - name: gitlab-runner