Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce disk pressure in workflows #1847

Merged
merged 9 commits into from
Jun 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,19 +70,22 @@ jobs:
if: always()
uses: ./.github/actions/save-logs

# Builds init packages since GoReleaser won't handle this for us
- name: Create release time CVE report
run: "make cve-report"

# Before we run GoReleaser we need to aggressively cleanup files to reduce disk pressure
- name: Cleanup files
run: |
sudo rm -rf zarf-sbom /tmp/zarf-*
lsblk -f
sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules
sudo env "PATH=$PATH" CI=true make delete-packages
sudo build/zarf tools clear-cache
sudo docker system prune --all
sudo docker system prune --all --force
sudo k3s crictl rmi --prune
go clean -cache
lsblk -f

# Builds init packages since GoReleaser won't handle this for us
- name: Create release time CVE report
run: "make cve-report"

# Set up AWS credentials for GoReleaser to upload backups of artifacts to S3
- name: Set AWS Credentials
uses: aws-actions/configure-aws-credentials@5727f247b64f324ec403ac56ae05e220fd02b65f # v2-node16
Expand Down
20 changes: 8 additions & 12 deletions .github/workflows/test-upgrade.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ concurrency:

jobs:
validate:
env:
# Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests
ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%"
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand All @@ -39,14 +42,6 @@ jobs:
with:
download-init-package: true

# Note we move the kubelet data to reduce the chance for disk pressure during the tests
- name: Move the kubelet root directory for 'k3s'
run: |
sudo mkdir -p /mnt/kubelet-data
echo >> zarf-config.toml
echo [package.deploy.set] >> zarf-config.toml
echo "k3s_args = '--disable traefik --kubelet-arg \"eviction-hard=imagefs.available<1%,nodefs.available<1%\" --kubelet-arg \"eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%\"'" >> zarf-config.toml

- name: Initialize the cluster with the release version
# NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
# in a previous step. This test run will the current release to create a K3s cluster.
Expand Down Expand Up @@ -80,18 +75,19 @@ jobs:
- name: "Cleanup after running tests"
# NOTE: This reduces disk pressure before the upgrade-specific tests begin
run: |
sudo lsblk -f
lsblk -f
sudo k3s crictl images
sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade
sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade

sudo k3s crictl rmi --prune
sudo rm -rf zarf-sbom /tmp/zarf-*/
sudo rm -rf zarf-sbom /tmp/zarf-*/ src/ui/node_modules
sudo env "PATH=$PATH" CI=true make delete-packages
sudo build/zarf tools clear-cache
sudo docker system prune --all --force
sudo k3s crictl rmi --prune

sudo lsblk -f
lsblk -f
sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes

- name: Run the upgrade tests
Expand Down
8 changes: 3 additions & 5 deletions .goreleaser.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,7 @@ builds:
# Save the built artifacts as binaries (instead of wrapping them in a tarball)
archives:
- format: binary
name_template: "{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}"
replacements:
darwin: Darwin
linux: Linux
windows: Windows
name_template: "{{ .ProjectName }}_{{ .Tag }}_{{- title .Os }}_{{ .Arch }}"

# generate a sha256 checksum of all release artifacts
# NOTE: We are explicitly adding the init-packages that are built prior to GoReleaser stage in the GitHub Actions workflow
Expand All @@ -40,6 +36,8 @@ checksum:
# generate sboms for each binary artifact
sboms:
- artifacts: binary
documents:
- "sbom_{{ .ProjectName }}_{{ .Tag }}_{{- title .Os }}_{{ .Arch }}.sbom"

snapshot:
name_template: "{{ incpatch .Version }}-snapshot"
Expand Down
3 changes: 3 additions & 0 deletions src/internal/cluster/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ func (c *Cluster) sanitizeZarfState(state types.ZarfState) types.ZarfState {
sanitizedState.RegistryInfo.PullPassword = "**sanitized**"
sanitizedState.RegistryInfo.Secret = "**sanitized**"

// Overwrite the ArtifactServer secret
sanitizedState.ArtifactServer.PushToken = "**sanitized**"

// Overwrite the Logging secret
sanitizedState.LoggingSecret = "**sanitized**"

Expand Down
45 changes: 33 additions & 12 deletions src/test/e2e/20_zarf_init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,15 @@ func TestZarfInit(t *testing.T) {
require.NoError(t, err)
}

// Check for any old secrets to ensure that they don't get saved in the init log
oldState := types.ZarfState{}
base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}")
if err == nil {
oldStateJSON, err := base64.StdEncoding.DecodeString(base64State)
require.NoError(t, err)
err = json.Unmarshal(oldStateJSON, &oldState)
}

// run `zarf init`
_, initStdErr, err := e2e.Zarf("init", "--components="+initComponents, "--nodeport", "31337", "-l", "trace", "--confirm")
require.NoError(t, err)
Expand All @@ -69,23 +78,19 @@ func TestZarfInit(t *testing.T) {
logText := e2e.GetLogFileContents(t, initStdErr)

// Verify that any state secrets were not included in the log
base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}")
state := types.ZarfState{}
base64State, _, err = e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}")
require.NoError(t, err)
stateJSON, err := base64.StdEncoding.DecodeString(base64State)
require.NoError(t, err)
state := types.ZarfState{}
err = json.Unmarshal(stateJSON, &state)
require.NoError(t, err)
require.NotContains(t, logText, state.AgentTLS.CA)
require.NotContains(t, logText, state.AgentTLS.Cert)
require.NotContains(t, logText, state.AgentTLS.Key)
require.NotContains(t, logText, state.ArtifactServer.PushToken)
require.NotContains(t, logText, state.GitServer.PullPassword)
require.NotContains(t, logText, state.GitServer.PushPassword)
require.NotContains(t, logText, state.RegistryInfo.PullPassword)
require.NotContains(t, logText, state.RegistryInfo.PushPassword)
require.NotContains(t, logText, state.RegistryInfo.Secret)
require.NotContains(t, logText, state.LoggingSecret)
checkLogForSensitiveState(t, logText, state)

// Check the old state values as well (if they exist) to ensure they weren't printed and then updated during init
if oldState.LoggingSecret != "" {
checkLogForSensitiveState(t, logText, oldState)
}

if e2e.ApplianceMode {
// make sure that we upgraded `k3s` correctly and are running the correct version - this should match that found in `packages/distros/k3s`
Expand All @@ -108,3 +113,19 @@ func TestZarfInit(t *testing.T) {
_, _, _ = e2e.Kubectl("scale", "deploy", "-n", "kube-system", "coredns", "--replicas=1")
_, _, _ = e2e.Kubectl("scale", "deploy", "-n", "zarf", "agent-hook", "--replicas=1")
}

func checkLogForSensitiveState(t *testing.T, logText string, zarfState types.ZarfState) {
require.NotContains(t, logText, zarfState.AgentTLS.CA)
require.NotContains(t, logText, string(zarfState.AgentTLS.CA))
require.NotContains(t, logText, zarfState.AgentTLS.Cert)
require.NotContains(t, logText, string(zarfState.AgentTLS.Cert))
require.NotContains(t, logText, zarfState.AgentTLS.Key)
require.NotContains(t, logText, string(zarfState.AgentTLS.Key))
require.NotContains(t, logText, zarfState.ArtifactServer.PushToken)
require.NotContains(t, logText, zarfState.GitServer.PullPassword)
require.NotContains(t, logText, zarfState.GitServer.PushPassword)
require.NotContains(t, logText, zarfState.RegistryInfo.PullPassword)
require.NotContains(t, logText, zarfState.RegistryInfo.PushPassword)
require.NotContains(t, logText, zarfState.RegistryInfo.Secret)
require.NotContains(t, logText, zarfState.LoggingSecret)
}
25 changes: 2 additions & 23 deletions src/test/packages/20-mismatched-arch-init/zarf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,5 @@ components:
import:
path: ../../../../packages/distros/k3s

# This package moves the injector & registries binaries
- name: zarf-injector
required: true
import:
path: ../../../../packages/zarf-registry

# Creates the temporary seed-registry
- name: zarf-seed-registry
required: true
import:
path: ../../../../packages/zarf-registry

# Creates the permanent registry
- name: zarf-registry
required: true
import:
path: ../../../../packages/zarf-registry

# Creates the pod+git mutating webhook
- name: zarf-agent
required: true
import:
path: ../../../../packages/zarf-agent
# This test package skips everything else since we explicitly expect 'expectedErrorMessage'
# to come from the `k3s` package or from Zarf checking an already existing cluster