From 47f909dc3d157d6e2e80445f6b133d320cab90eb Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 18:29:22 -0500 Subject: [PATCH 1/9] Reduce disk pressure in workflows --- .github/workflows/release.yml | 15 +++++++++------ .github/workflows/test-upgrade.yml | 20 ++++++++------------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81f7da4b72..9985db6efe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,19 +70,22 @@ jobs: if: always() uses: ./.github/actions/save-logs + # Builds init packages since GoReleaser won't handle this for us + - name: Create release time CVE report + run: "make cve-report" + + # Before we run goreleaser we need to aggressively cleanup files to reduce disk pressure - name: Cleanup files run: | - sudo rm -rf zarf-sbom /tmp/zarf-* + lsblk -f + sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules sudo env "PATH=$PATH" CI=true make delete-packages sudo build/zarf tools clear-cache - sudo docker system prune --all + sudo docker system prune --all --force + sudo k3s crictl rmi --prune go clean -cache lsblk -f - # Builds init packages since GoReleaser won't handle this for us - - name: Create release time CVE report - run: "make cve-report" - # Set up AWS credentials for GoReleaser to upload backups of artifacts to S3 - name: Set AWS Credentials uses: aws-actions/configure-aws-credentials@5727f247b64f324ec403ac56ae05e220fd02b65f # v2-node16 diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index a365b5aa7d..9594e6eaed 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -23,6 +23,9 @@ concurrency: jobs: validate: + env: + # Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests + ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%" runs-on: ubuntu-latest steps: - name: Checkout @@ -39,14 +42,6 @@ jobs: with: download-init-package: true - # Note we move the kubelet data to reduce the chance for disk pressure during the tests - - name: Move the kubelet root directory for 'k3s' - run: | - sudo mkdir -p /mnt/kubelet-data - echo >> zarf-config.toml - echo [package.deploy.set] >> zarf-config.toml - echo "k3s_args = '--disable traefik --kubelet-arg \"eviction-hard=imagefs.available<1%,nodefs.available<1%\" --kubelet-arg \"eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%\"'" >> zarf-config.toml - - name: Initialize the cluster with the release version # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed # in a previous step. This test run will the current release to create a K3s cluster. @@ -80,18 +75,19 @@ jobs: - name: "Cleanup after running tests" # NOTE: This reduces disk pressure before the upgrade-specific tests begin run: | - sudo lsblk -f + lsblk -f sudo k3s crictl images sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade - sudo k3s crictl rmi --prune - sudo rm -rf zarf-sbom /tmp/zarf-*/ + sudo rm -rf zarf-sbom /tmp/zarf-*/ src/ui/node_modules sudo env "PATH=$PATH" CI=true make delete-packages sudo build/zarf tools clear-cache + sudo docker system prune --all --force + sudo k3s crictl rmi --prune - sudo lsblk -f + lsblk -f sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes - name: Run the upgrade tests From 69f31efda2c9dafef61c2468ca9fc3582ade941e Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 18:33:08 -0500 Subject: [PATCH 2/9] Add capitalization to GoReleaser --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9985db6efe..357dd038c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,7 +74,7 @@ jobs: - name: Create release time CVE report run: "make cve-report" - # Before we run goreleaser we need to aggressively cleanup files to reduce disk pressure + # Before we run GoReleaser we need to aggressively cleanup files to reduce disk pressure - name: Cleanup files run: | lsblk -f From d2d0d31ad8dab2f0426de7f6bd17020129bcbd5a Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 18:36:28 -0500 Subject: [PATCH 3/9] Update deprecated 'replacements' key --- .goreleaser.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 828e9ed462..da16a7f1b5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -23,11 +23,7 @@ builds: # Save the built artifacts as binaries (instead of wrapping them in a tarball) archives: - format: binary - name_template: "{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}" - replacements: - darwin: Darwin - linux: Linux - windows: Windows + name_template: "{{ .ProjectName }}_{{ .Tag }}_{{- title .Os }}_{{ .Arch }}" # generate a sha256 checksum of all release artifacts # NOTE: We are explicitly adding the init-packages that are built prior to GoReleaser stage in the GitHub Actions workflow From 74eade057b7ecb0e0de8115526f24a1c337bb79a Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 18:40:19 -0500 Subject: [PATCH 4/9] More closely match Binary name in SBOM format and prefix with 'sbom' --- .goreleaser.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index da16a7f1b5..ada0dd1ee2 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -36,6 +36,8 @@ checksum: # generate sboms for each binary artifact sboms: - artifacts: binary + documents: + - "sbom_{{ .ProjectName }}_{{ .Tag }}_{{- title .Os }}_{{ .Arch }}.sbom" snapshot: name_template: "{{ incpatch .Version }}-snapshot" From a395cf7a7a1da8a053c6943d8010bc1608f503a4 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 19:05:43 -0500 Subject: [PATCH 5/9] Fixup state sanitization --- src/internal/cluster/state.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/internal/cluster/state.go b/src/internal/cluster/state.go index a39f903e15..da5bdcee1d 100644 --- a/src/internal/cluster/state.go +++ b/src/internal/cluster/state.go @@ -183,6 +183,9 @@ func (c *Cluster) sanitizeZarfState(state types.ZarfState) types.ZarfState { sanitizedState.RegistryInfo.PullPassword = "**sanitized**" sanitizedState.RegistryInfo.Secret = "**sanitized**" + // Overwrite the ArtifactServer secret + sanitizedState.ArtifactServer.PushToken = "**sanitized**" + // Overwrite the Logging secret sanitizedState.LoggingSecret = "**sanitized**" From 75cf556d0a5fb3cea0384034cc8251eb50cf651c Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 19:12:26 -0500 Subject: [PATCH 6/9] Validate that the tests catch old state --- src/internal/cluster/state.go | 2 +- src/test/e2e/20_zarf_init_test.go | 42 ++++++++++++++++++++++--------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/internal/cluster/state.go b/src/internal/cluster/state.go index da5bdcee1d..e05e5f073e 100644 --- a/src/internal/cluster/state.go +++ b/src/internal/cluster/state.go @@ -184,7 +184,7 @@ func (c *Cluster) sanitizeZarfState(state types.ZarfState) types.ZarfState { sanitizedState.RegistryInfo.Secret = "**sanitized**" // Overwrite the ArtifactServer secret - sanitizedState.ArtifactServer.PushToken = "**sanitized**" + // sanitizedState.ArtifactServer.PushToken = "**sanitized**" // Overwrite the Logging secret sanitizedState.LoggingSecret = "**sanitized**" diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index d45606105d..9c8b7bc8dd 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -61,6 +61,15 @@ func TestZarfInit(t *testing.T) { require.NoError(t, err) } + // Check for any old secrets to ensure that they don't get saved in the init log + oldState := types.ZarfState{} + base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}") + if err != nil { + oldStateJSON, err := base64.StdEncoding.DecodeString(base64State) + require.NoError(t, err) + err = json.Unmarshal(oldStateJSON, &oldState) + } + // run `zarf init` _, initStdErr, err := e2e.Zarf("init", "--components="+initComponents, "--nodeport", "31337", "-l", "trace", "--confirm") require.NoError(t, err) @@ -69,23 +78,19 @@ func TestZarfInit(t *testing.T) { logText := e2e.GetLogFileContents(t, initStdErr) // Verify that any state secrets were not included in the log - base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}") + state := types.ZarfState{} + base64State, _, err = e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}") require.NoError(t, err) stateJSON, err := base64.StdEncoding.DecodeString(base64State) require.NoError(t, err) - state := types.ZarfState{} err = json.Unmarshal(stateJSON, &state) require.NoError(t, err) - require.NotContains(t, logText, state.AgentTLS.CA) - require.NotContains(t, logText, state.AgentTLS.Cert) - require.NotContains(t, logText, state.AgentTLS.Key) - require.NotContains(t, logText, state.ArtifactServer.PushToken) - require.NotContains(t, logText, state.GitServer.PullPassword) - require.NotContains(t, logText, state.GitServer.PushPassword) - require.NotContains(t, logText, state.RegistryInfo.PullPassword) - require.NotContains(t, logText, state.RegistryInfo.PushPassword) - require.NotContains(t, logText, state.RegistryInfo.Secret) - require.NotContains(t, logText, state.LoggingSecret) + checkLogForSensitiveState(t, logText, state) + + // Check the old state values as well (if they exist) to ensure they weren't printed and then updated during init + if oldState.LoggingSecret != "" { + checkLogForSensitiveState(t, logText, oldState) + } if e2e.ApplianceMode { // make sure that we upgraded `k3s` correctly and are running the correct version - this should match that found in `packages/distros/k3s` @@ -108,3 +113,16 @@ func TestZarfInit(t *testing.T) { _, _, _ = e2e.Kubectl("scale", "deploy", "-n", "kube-system", "coredns", "--replicas=1") _, _, _ = e2e.Kubectl("scale", "deploy", "-n", "zarf", "agent-hook", "--replicas=1") } + +func checkLogForSensitiveState(t *testing.T, logText string, zarfState types.ZarfState) { + require.NotContains(t, logText, zarfState.AgentTLS.CA) + require.NotContains(t, logText, zarfState.AgentTLS.Cert) + require.NotContains(t, logText, zarfState.AgentTLS.Key) + require.NotContains(t, logText, zarfState.ArtifactServer.PushToken) + require.NotContains(t, logText, zarfState.GitServer.PullPassword) + require.NotContains(t, logText, zarfState.GitServer.PushPassword) + require.NotContains(t, logText, zarfState.RegistryInfo.PullPassword) + require.NotContains(t, logText, zarfState.RegistryInfo.PushPassword) + require.NotContains(t, logText, zarfState.RegistryInfo.Secret) + require.NotContains(t, logText, zarfState.LoggingSecret) +} From fddec6abf22b6f39a323216a5db4694478f83bfd Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 19:54:57 -0500 Subject: [PATCH 7/9] Improve init sensitive test --- src/test/e2e/20_zarf_init_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 9c8b7bc8dd..31839db420 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -64,7 +64,7 @@ func TestZarfInit(t *testing.T) { // Check for any old secrets to ensure that they don't get saved in the init log oldState := types.ZarfState{} base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}") - if err != nil { + if err == nil { oldStateJSON, err := base64.StdEncoding.DecodeString(base64State) require.NoError(t, err) err = json.Unmarshal(oldStateJSON, &oldState) @@ -116,8 +116,11 @@ func TestZarfInit(t *testing.T) { func checkLogForSensitiveState(t *testing.T, logText string, zarfState types.ZarfState) { require.NotContains(t, logText, zarfState.AgentTLS.CA) + require.NotContains(t, logText, string(zarfState.AgentTLS.CA)) require.NotContains(t, logText, zarfState.AgentTLS.Cert) + require.NotContains(t, logText, string(zarfState.AgentTLS.Cert)) require.NotContains(t, logText, zarfState.AgentTLS.Key) + require.NotContains(t, logText, string(zarfState.AgentTLS.Key)) require.NotContains(t, logText, zarfState.ArtifactServer.PushToken) require.NotContains(t, logText, zarfState.GitServer.PullPassword) require.NotContains(t, logText, zarfState.GitServer.PushPassword) From cc64d03c2a6e3dd841bdeebedbaaf30f8cac3890 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 19:59:44 -0500 Subject: [PATCH 8/9] Properly sanitize the artifact push token --- src/internal/cluster/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/cluster/state.go b/src/internal/cluster/state.go index e05e5f073e..da5bdcee1d 100644 --- a/src/internal/cluster/state.go +++ b/src/internal/cluster/state.go @@ -184,7 +184,7 @@ func (c *Cluster) sanitizeZarfState(state types.ZarfState) types.ZarfState { sanitizedState.RegistryInfo.Secret = "**sanitized**" // Overwrite the ArtifactServer secret - // sanitizedState.ArtifactServer.PushToken = "**sanitized**" + sanitizedState.ArtifactServer.PushToken = "**sanitized**" // Overwrite the Logging secret sanitizedState.LoggingSecret = "**sanitized**" From d726f7cbdd0c710d9eb5352719739f4cd5e81a3f Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Sat, 24 Jun 2023 20:14:15 -0500 Subject: [PATCH 9/9] Simplify the zarf init mismatch test --- .../20-mismatched-arch-init/zarf.yaml | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/src/test/packages/20-mismatched-arch-init/zarf.yaml b/src/test/packages/20-mismatched-arch-init/zarf.yaml index ba41af4f52..eb2bef901b 100644 --- a/src/test/packages/20-mismatched-arch-init/zarf.yaml +++ b/src/test/packages/20-mismatched-arch-init/zarf.yaml @@ -8,26 +8,5 @@ components: import: path: ../../../../packages/distros/k3s - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - path: ../../../../packages/zarf-registry - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - path: ../../../../packages/zarf-registry - - # Creates the permanent registry - - name: zarf-registry - required: true - import: - path: ../../../../packages/zarf-registry - - # Creates the pod+git mutating webhook - - name: zarf-agent - required: true - import: - path: ../../../../packages/zarf-agent + # This test package skips everything else since we explicitly expect 'expectedErrorMessage' + # to come from the `k3s` package or from Zarf checking an already existing cluster