From a79bcded88b8e7ac95c2578c8d2502f76c140179 Mon Sep 17 00:00:00 2001 From: Jonathan Perry Date: Thu, 29 Sep 2022 17:28:13 -0400 Subject: [PATCH] Feature: Support Using Zarf With an External Registry and Repository (#754) ## Description This PR introduces the ability to connect to an already existing (and reachable) Container Registry and/or Git Repository during the `zarf init` command. Closes #570 (Support using an external git server) Closes #560 (Support using an external registry) This implementation will serve as a good midway point on having a fully HA in-cluster registry #375. ## PR Feature List - Added several flags to the `init` command to support using an external git repository - Added several flags to the `init` command to support using an external container registry - Update `zarf connect registry` to direct to `{HOST}/v2/_catalog` (this was confusing some other people since it would originally seem like the registry was returning an empty page) - Add utility function to create a tunnel to a service URL - Created slightly better regexp for replacing the host from a `containerImage` url - semi-refactored the `zarf package deploy` logic ## Breaking Changes List - We are changing the structure of the names of repos & containers we are pushing (we are simplifying the name and adding a sha1 hash of the original name to the end of the name) Co-authored-by: Wayne Starr Co-authored-by: Megamind <882485+jeff-mccoy@users.noreply.github.com> --- .github/workflows/test-external.yml | 73 +++ Makefile | 23 +- .../100-cli-commands/zarf_init.md | 46 +- .../zarf_prepare_patch-git.md | 3 +- .../100-cli-commands/zarf_tools.md | 2 +- .../zarf_tools_get-git-password.md | 30 + examples/flux-test/zarf.yaml | 2 +- go.mod | 4 - go.sum | 21 - packages/gitea/gitea-values.yaml | 2 +- packages/zarf-agent/manifests/deployment.yaml | 6 + packages/zarf-registry/connect.yaml | 3 +- src/cmd/destroy.go | 2 +- src/cmd/initialize.go | 54 +- src/cmd/prepare.go | 4 +- src/cmd/tools.go | 8 +- src/config/config.go | 47 +- src/config/secret.go | 74 --- src/internal/agent/hooks/flux.go | 61 +- src/internal/agent/hooks/pods.go | 52 +- src/internal/git/pull.go | 12 +- src/internal/git/push.go | 87 ++- src/internal/git/utils.go | 65 ++- src/internal/helm/post-render.go | 4 +- src/internal/images/push.go | 42 +- src/internal/k8s/secrets.go | 12 +- src/internal/k8s/tunnel.go | 35 ++ src/internal/message/message.go | 6 + src/internal/packager/components.go | 2 +- src/internal/packager/create.go | 5 +- src/internal/packager/deploy.go | 519 +++++++++++------- src/internal/packager/injector.go | 30 +- src/internal/packager/seed.go | 123 ++++- src/internal/template/template.go | 21 +- src/internal/utils/image.go | 64 ++- src/internal/utils/network.go | 18 + src/test/e2e/22_git_and_flux_test.go | 14 +- src/test/e2e/common.go | 2 +- src/test/e2e/main_test.go | 2 +- src/test/external-test/README.md | 26 + .../external-test/docker-registry-values.yaml | 19 + src/test/external-test/external_init_test.go | 92 ++++ src/test/external-test/gitea-values.yaml | 34 ++ src/test/external-test/secret.yaml | 10 + src/types/k8s.go | 41 +- src/types/runtime.go | 39 +- src/ui/lib/api-types.ts | 184 ++++++- zarf.yaml | 3 + 48 files changed, 1505 insertions(+), 523 deletions(-) create mode 100644 .github/workflows/test-external.yml create mode 100644 docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_get-git-password.md delete mode 100644 src/config/secret.go create mode 100644 src/test/external-test/README.md create mode 100644 src/test/external-test/docker-registry-values.yaml create mode 100644 src/test/external-test/external_init_test.go create mode 100644 src/test/external-test/gitea-values.yaml create mode 100644 src/test/external-test/secret.yaml diff --git a/.github/workflows/test-external.yml b/.github/workflows/test-external.yml new file mode 100644 index 0000000000..dac8e91039 --- /dev/null +++ b/.github/workflows/test-external.yml @@ -0,0 +1,73 @@ +name: test-external +on: + pull_request: + paths-ignore: + - "**.md" + - "**.jpg" + - "**.png" + - "**.gif" + - "**.svg" + - "adr/**" + - "docs/**" + +# Abort prior jobs in the same workflow / PR +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: "Dependency: Install Golang" + uses: actions/setup-go@v3 + with: + go-version: 1.19.x + + - name: "Dependency: Install Docker Buildx" + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: "Dependency: Install K3d" + run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | sudo bash" + + - name: "Dependency: K3d cluster init" + run: k3d cluster delete && k3d cluster create + + - name: "Dependency: Install Helm" + run: "curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash" + + - name: "Install Kubectl" + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + chmod +x kubectl + mv ./kubectl /usr/local/bin/kubectl + + - name: "Checkout Repo" + uses: actions/checkout@v3 + + - name: "Build CLI" + run: make build-cli-linux-amd ARCH=amd64 + + - name: "Zarf Agent: Login to Docker Hub" + uses: docker/login-action@v2 + with: + username: zarfdev + password: ${{ secrets.ZARF_DEV_DOCKERHUB }} + + - name: "Zarf Agent: Build and Publish the Image" + run: | + cp build/zarf build/zarf-linux-amd64 + docker buildx build --push --platform linux/amd64 --tag zarfdev/agent:$GITHUB_SHA . + + - name: "Make Init Package" + run: make init-package AGENT_IMAGE=zarfdev/agent:$GITHUB_SHA + + - name: "Run Tests" + # NOTE: This test run will create its own K3d cluster. A single cluster will be used throughout the test run. + run: make test-external + + - name: "Cleanup" + run: make destroy diff --git a/Makefile b/Makefile index a6df12401c..7cc1289c92 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ ifneq ($(UNAME_S),Linux) endif endif -AGENT_IMAGE ?= defenseunicorns/zarf-agent:v0.21.1 +AGENT_IMAGE ?= zarfdev/agent:a57bb136f21441c66630403412c6f03fc7f9cd49 CLI_VERSION := $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion") BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)' @@ -105,19 +105,20 @@ cve-report: go run main.go tools sbom packages . -o json | grype -o template -t .hooks/grype.tmpl > build/zarf-known-cves.csv # Inject and deploy a new dev version of zarf agent for testing (should have an existing zarf agent deployemt) -# @todo: find a clean way to support Kind or k3d: k3d image import $(tag) +# @todo: find a clean way to dynamically support Kind or k3d: +# when using kind: kind load docker-image $(tag) +# when using k3d: k3d image import $(tag) dev-agent-image: $(eval tag := defenseunicorns/dev-zarf-agent:$(shell date +%s)) $(eval arch := $(shell uname -m)) CGO_ENABLED=0 GOOS=linux go build -o build/zarf-linux-$(arch) main.go DOCKER_BUILDKIT=1 docker build --tag $(tag) --build-arg TARGETARCH=$(arch) . && \ - kind load docker-image zarf-agent:$(tag) && \ + k3d image import $(tag) && \ kubectl -n zarf set image deployment/agent-hook server=$(tag) init-package: ## Create the zarf init package, macos "brew install coreutils" first @test -s $(ZARF_BIN) || $(MAKE) build-cli - - @test -s ./build/zarf-init-$(ARCH).tar.zst || $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm . + $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm . ci-release: init-package ## Create the init package @@ -140,10 +141,18 @@ build-examples: @test -s ./build/zarf-package-compose-example-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/composable-packages -o build -a $(ARCH) --confirm - @test -s ./build/zarf-package-flux-test-${ARCH}.tar.zst || $(ZARF_BIN) package create examples/flux-test -o build -a $(ARCH) --confirm + @test -s ./build/zarf-package-flux-test-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/flux-test -o build -a $(ARCH) --confirm ## Run e2e tests. Will automatically build any required dependencies that aren't present. ## Requires an existing cluster for the env var APPLIANCE_MODE=true .PHONY: test-e2e -test-e2e: init-package build-examples +test-e2e: build-examples + @test -s ./build/zarf-init-$(ARCH).tar.zst || $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm . + @test -s ./build/zarf-init-$(ARCH).tar.zst || $(MAKE) init-package cd src/test/e2e && go test -failfast -v -timeout 30m + +test-external: + @test -s $(ZARF_BIN) || $(MAKE) build-cli + @test -s ./build/zarf-init-$(ARCH).tar.zst || $(MAKE) init-package + @test -s ./build/zarf-package-flux-test-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/flux-test -o build -a $(ARCH) --confirm + cd src/test/external-test && go test -failfast -v -timeout 30m diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md index c77744853a..3ad8be56e6 100644 --- a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md @@ -10,6 +10,28 @@ If you do not have a k8s cluster already configured, this command will give you This command looks for a zarf-init package in the local directory that the command was executed from. If no package is found in the local directory and the Zarf CLI exists somewhere outside of the current directory, Zarf will failover and attempt to find a zarf-init package in the directory that the Zarf binary is located in. + +Example Usage: +# Initializing without any optional components: +zarf init + +# Initializing w/ Zarfs internal git server: +zarf init --components=git-server + +# Initializing w/ Zarfs internal git server and PLG stack: +zarf init --components=git-server,logging + +# Initializing w/ an internal registry but with a different nodeport: +zarf init --nodeport=30333 + +# Initializing w/ an external registry: +zarf init --registry-push-password={PASSWORD} --registry-push-username={USERNAME} --registry-url={URL} + +# Initializing w/ an external git server: +zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-url={URL} + + + ``` zarf init [flags] ``` @@ -17,13 +39,23 @@ zarf init [flags] ### Options ``` - --components string Comma-separated list of components to install. - --confirm Confirm the install without prompting - -h, --help help for init - --nodeport string Nodeport to access the Zarf container registry. Between [30000-32767] - --secret string Root secret value that is used to 'seed' other secrets - --storage-class string Describe the StorageClass to be used - --tmpdir string Specify the temporary directory to use for intermediate files + --components string Comma-separated list of components to install. + --confirm Confirm the install without prompting + --git-pull-password string Password for the pull-only user to access the git server + --git-pull-username string Username for pull-only access to the git server + --git-push-password string Password for the push-user to access the git server + --git-push-username string Username to access to the git server Zarf is configured to use. User must be able to create repositories via 'git push' (default "zarf-git-user") + --git-url string External git server url to use for this Zarf cluster + -h, --help help for init + --nodeport int Nodeport to access a registry internal to the k8s cluster. Between [30000-32767] + --registry-pull-password string Password for the pull-only user to access the registry + --registry-pull-username string Username for pull-only access to the registry + --registry-push-password string Password for the push-user to connect to the registry + --registry-push-username string Username to access to the registry Zarf is configured to use (default "zarf-push") + --registry-secret string Registry secret value + --registry-url string External registry url address to use for this Zarf cluster + --storage-class string Describe the StorageClass to be used + --tmpdir string Specify the temporary directory to use for intermediate files ``` ### Options inherited from parent commands diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md index 55c40a17d0..5b38f04329 100644 --- a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md @@ -10,7 +10,8 @@ zarf prepare patch-git [HOST] [FILE] [flags] ### Options ``` - -h, --help help for patch-git + --git-account string User or organization name for the git account that the repos are created under. (default "zarf-git-user") + -h, --help help for patch-git ``` ### Options inherited from parent commands diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md index 342fba7d6d..253ebd3e12 100644 --- a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md @@ -20,7 +20,7 @@ Collection of additional tools to make airgap easier * [zarf](zarf.md) - DevSecOps Airgap Toolkit * [zarf tools archiver](zarf_tools_archiver.md) - Compress/Decompress tools for Zarf packages -* [zarf tools get-admin-password](zarf_tools_get-admin-password.md) - Returns the Zarf admin password for gitea read from the zarf-state secret in the zarf namespace +* [zarf tools get-git-password](zarf_tools_get-git-password.md) - Returns the push user's password for the Git server * [zarf tools monitor](zarf_tools_monitor.md) - Launch K9s tool for managing K8s clusters * [zarf tools registry](zarf_tools_registry.md) - Collection of registry commands provided by Crane * [zarf tools sbom](zarf_tools_sbom.md) - SBOM tools provided by Anchore Syft diff --git a/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_get-git-password.md b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_get-git-password.md new file mode 100644 index 0000000000..aa297585c1 --- /dev/null +++ b/docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_get-git-password.md @@ -0,0 +1,30 @@ +## zarf tools get-git-password + +Returns the push user's password for the Git server + +### Synopsis + +Reads the password for a user with push access to the configured Git server from the zarf-state secret in the zarf namespace + +``` +zarf tools get-git-password [flags] +``` + +### Options + +``` + -h, --help help for get-git-password +``` + +### Options inherited from parent commands + +``` + -a, --architecture string Architecture for OCI images + -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace + --no-progress Disable fancy UI progress bars, spinners, logos, etc. +``` + +### SEE ALSO + +* [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier + diff --git a/examples/flux-test/zarf.yaml b/examples/flux-test/zarf.yaml index 9942f398fe..04ca68f345 100644 --- a/examples/flux-test/zarf.yaml +++ b/examples/flux-test/zarf.yaml @@ -26,6 +26,6 @@ components: - podinfo-source.yaml - podinfo-kustomization.yaml repos: - - https://github.com/stefanprodan/podinfo + - https://github.com/stefanprodan/podinfo.git images: - ghcr.io/stefanprodan/podinfo:6.1.6 diff --git a/go.mod b/go.mod index 7eee84908d..d5afb1ab61 100644 --- a/go.mod +++ b/go.mod @@ -372,11 +372,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.design/x/clipboard v0.6.2 // indirect golang.org/x/exp v0.0.0-20220823124025-807a23277127 // indirect - golang.org/x/exp/shiny v0.0.0-20220921164117-439092de6870 // indirect - golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect - golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect diff --git a/go.sum b/go.sum index eddb0d033c..7cffb8a979 100644 --- a/go.sum +++ b/go.sum @@ -657,8 +657,6 @@ github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCF github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= -github.com/derailed/k9s v0.26.4 h1:kmYrdhBAMFf38kP/W3vgBE/szR/opK0+u0TY/SL6VuQ= -github.com/derailed/k9s v0.26.4/go.mod h1:fEfXpctbj4HyXEvD6+P+8lL94Q6eiojofr/i6A9Zus8= github.com/derailed/k9s v0.26.5 h1:afOQSIad+893o9EjzYN3DMQ2o4Cjj+JraBfLep/GKEw= github.com/derailed/k9s v0.26.5/go.mod h1:uUX//U+7n6KYzdmqj4QLYdXdu2akfSP81WxDyeo/ZuY= github.com/derailed/k9s v0.26.6 h1://neH2SAF9akWkKKrfZvRVIg/Z1W/TLMs/k0T0aUBPU= @@ -2315,8 +2313,6 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= -golang.design/x/clipboard v0.6.2 h1:a3Np4qfKnLWwfFJQhUWU3IDeRfmVuqWl+QPtP4CSYGw= -golang.design/x/clipboard v0.6.2/go.mod h1:kqBSweBP0/im4SZGGjLrppH0D400Hnfo5WbFKSNK8N4= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2362,8 +2358,6 @@ golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY= -golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7 h1:WJywXQVIb56P2kAvXeMGTIgQ1ZHQxR60+F9dLsodECc= golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= @@ -2371,7 +2365,6 @@ golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -2382,12 +2375,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20220823124025-807a23277127 h1:S4NrSKDfihhl3+4jSTgwoIevKxX9p7Iv9x++OEIptDo= golang.org/x/exp v0.0.0-20220823124025-807a23277127/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp/shiny v0.0.0-20220921164117-439092de6870 h1:GjCs9zNN8fojJskeK7QiiVecCaMk0dfGTyL6IUcmp0o= -golang.org/x/exp/shiny v0.0.0-20220921164117-439092de6870/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -2402,8 +2391,6 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 h1:3In5TnfvnuXTF/uflgpYxSCEGP2NdYT37KsPh3VjZYU= -golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554/go.mod h1:jFTmtFYCV0MFtXBU+J5V/+5AUeVS0ON/0WkE/KSrl6E= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -3175,8 +3162,6 @@ honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.25.1 h1:yL7du50yc93k17nH/Xe9jujAYrcDkI/i5DL1jPz4E3M= -k8s.io/api v0.25.1/go.mod h1:hh4itDvrWSJsmeUc28rIFNri8MatNAAxJjKcQmhX6TU= k8s.io/api v0.25.2 h1:v6G8RyFcwf0HR5jQGIAYlvtRNrxMJQG1xJzaSeVnIS8= k8s.io/api v0.25.2/go.mod h1:qP1Rn4sCVFwx/xIhe+we2cwBLTXNcheRyYXwajonhy0= k8s.io/apiextensions-apiserver v0.25.1 h1:HEIKlxj6oHaDwHgotEIX/Ld5K/RGuOFwN/TWMiQ5s5s= @@ -3184,8 +3169,6 @@ k8s.io/apiextensions-apiserver v0.25.1/go.mod h1:67sgnMs2yIO2iV4DpCdS91vlP+pdnVI k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.25.1 h1:t0XrnmCEHVgJlR2arwO8Awp9ylluDic706WePaYCBTI= -k8s.io/apimachinery v0.25.1/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA= k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs= k8s.io/apimachinery v0.25.2/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= @@ -3198,8 +3181,6 @@ k8s.io/cli-runtime v0.25.1/go.mod h1:JSzAcqIK3JK7Ab/TY0PENKhmEg/HboNWK3VKiwsYB6E k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.25.1 h1:uFj4AJKtE1/ckcSKz8IhgAuZTdRXZDKev8g387ndD58= -k8s.io/client-go v0.25.1/go.mod h1:rdFWTLV/uj2C74zGbQzOsmXPUtMAjSf7ajil4iJUNKo= k8s.io/client-go v0.25.2 h1:SUPp9p5CwM0yXGQrwYurw9LWz+YtMwhWd0GqOsSiefo= k8s.io/client-go v0.25.2/go.mod h1:i7cNU7N+yGQmJkewcRD2+Vuj4iz7b30kI8OcL3horQ4= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= @@ -3225,8 +3206,6 @@ k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/metrics v0.25.1 h1:cp9WcR3PAN8xx5kBlbWCQQfkFwacjhKhITZafBJfIGs= k8s.io/metrics v0.25.1/go.mod h1:/t3eughLPd1sQNc47py2vTOY8e1E8bIxecA8rq/qQjM= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220922133306-665eaaec4324 h1:i+xdFemcSNuJvIfBlaYuXgRondKxK4z4prVPKzEaelI= k8s.io/utils v0.0.0-20220922133306-665eaaec4324/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index 1ce5a3b8df..69a6cdbe5a 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -2,7 +2,7 @@ persistence: storageClass: "###ZARF_STORAGE_CLASS###" gitea: admin: - username: "zarf-git-user" + username: "###ZARF_GIT_PUSH###" password: "###ZARF_GIT_AUTH_PUSH###" email: "zarf@localhost" config: diff --git a/packages/zarf-agent/manifests/deployment.yaml b/packages/zarf-agent/manifests/deployment.yaml index ce907ddf02..ddbc3e69e4 100644 --- a/packages/zarf-agent/manifests/deployment.yaml +++ b/packages/zarf-agent/manifests/deployment.yaml @@ -41,7 +41,13 @@ spec: - name: tls-certs mountPath: /etc/certs readOnly: true + - name: zarf-state + mountPath: /etc/zarf-state + readOnly: true volumes: - name: tls-certs secret: secretName: agent-hook-tls + - name: zarf-state + secret: + secretName: zarf-state diff --git a/packages/zarf-registry/connect.yaml b/packages/zarf-registry/connect.yaml index 4badda3512..d89465b989 100644 --- a/packages/zarf-registry/connect.yaml +++ b/packages/zarf-registry/connect.yaml @@ -4,9 +4,10 @@ metadata: name: zarf-connect-registry labels: # Enables "zarf connect registry" - zarf.dev/connect-name: registry + zarf.dev/connect-name: registry annotations: zarf.dev/connect-description: "Internal Zarf Registry (run zarf tools registry login to authenticate)" + zarf.dev/connect-url: "/v2/_catalog" spec: ports: - port: 5000 diff --git a/src/cmd/destroy.go b/src/cmd/destroy.go index f21865a90b..8b191e99bd 100644 --- a/src/cmd/destroy.go +++ b/src/cmd/destroy.go @@ -43,7 +43,7 @@ var destroyCmd = &cobra.Command{ } // If Zarf deployed the cluster, burn it all down - if state.ZarfAppliance || (state.Secret == "") { + if state.ZarfAppliance || (state.Distro == "") { // Check if we have the scripts to destory everything fileInfo, err := os.Stat(config.ZarfCleanupScriptsPath) if errors.Is(err, os.ErrNotExist) || !fileInfo.IsDir() { diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index d21d762caf..2e3bc6f68c 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -32,12 +32,25 @@ var initCmd = &cobra.Command{ "This command looks for a zarf-init package in the local directory that the command was executed " + "from. If no package is found in the local directory and the Zarf CLI exists somewhere outside of " + "the current directory, Zarf will failover and attempt to find a zarf-init package in the directory " + - "that the Zarf binary is located in.\n", + "that the Zarf binary is located in.\n\n\n\n" + + + "Example Usage:\n" + + "# Initializing without any optional components:\nzarf init\n\n" + + "# Initializing w/ Zarfs internal git server:\nzarf init --components=git-server\n\n" + + "# Initializing w/ Zarfs internal git server and PLG stack:\nzarf init --components=git-server,logging\n\n" + + "# Initializing w/ an internal registry but with a different nodeport:\nzarf init --nodeport=30333\n\n" + + "# Initializing w/ an external registry:\nzarf init --registry-push-password={PASSWORD} --registry-push-username={USERNAME} --registry-url={URL}\n\n" + + "# Initializing w/ an external git server:\nzarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-url={URL}\n\n", Run: func(cmd *cobra.Command, args []string) { zarfLogo := message.GetLogo() _, _ = fmt.Fprintln(os.Stderr, zarfLogo) + err := validateInitFlags() + if err != nil { + message.Fatal(err, "Invalid command flags were provided.") + } + // Continue running package deploy for all components like any other package initPackageName := fmt.Sprintf("zarf-init-%s.tar.zst", config.GetArch()) config.DeployOptions.PackagePath = initPackageName @@ -108,12 +121,45 @@ var initCmd = &cobra.Command{ }, } +func validateInitFlags() error { + // If 'git-url' is provided, make sure they provided values for the username and password of the push user + if config.InitOptions.GitServer.Address != "" { + if config.InitOptions.GitServer.PushUsername == "" || config.InitOptions.GitServer.PushPassword == "" { + return fmt.Errorf("the 'git-push-username' and 'git-push-password' flags must be provided if the 'git-url' flag is provided") + } + } + + //If 'registry-url' is provided, make sure they provided values for the username and password of the push user + if config.InitOptions.RegistryInfo.Address != "" { + if config.InitOptions.RegistryInfo.PushUsername == "" || config.InitOptions.RegistryInfo.PushPassword == "" { + return fmt.Errorf("the 'registry-push-username' and 'registry-push-password' flags must be provided if the 'registry-url' flag is provided ") + } + } + return nil +} + func init() { rootCmd.AddCommand(initCmd) initCmd.Flags().BoolVar(&config.CommonOptions.Confirm, "confirm", false, "Confirm the install without prompting") initCmd.Flags().StringVar(&config.CommonOptions.TempDirectory, "tmpdir", "", "Specify the temporary directory to use for intermediate files") initCmd.Flags().StringVar(&config.DeployOptions.Components, "components", "", "Comma-separated list of components to install.") - initCmd.Flags().StringVar(&config.DeployOptions.StorageClass, "storage-class", "", "Describe the StorageClass to be used") - initCmd.Flags().StringVar(&config.DeployOptions.Secret, "secret", "", "Root secret value that is used to 'seed' other secrets") - initCmd.Flags().StringVar(&config.DeployOptions.NodePort, "nodeport", "", "Nodeport to access the Zarf container registry. Between [30000-32767]") + initCmd.Flags().StringVar(&config.InitOptions.StorageClass, "storage-class", "", "Describe the StorageClass to be used") + + // Flags for using an external Git server + initCmd.Flags().StringVar(&config.InitOptions.GitServer.Address, "git-url", "", "External git server url to use for this Zarf cluster") + initCmd.Flags().StringVar(&config.InitOptions.GitServer.PushUsername, "git-push-username", config.ZarfGitPushUser, "Username to access to the git server Zarf is configured to use. User must be able to create repositories via 'git push'") + initCmd.Flags().StringVar(&config.InitOptions.GitServer.PushPassword, "git-push-password", "", "Password for the push-user to access the git server") + initCmd.Flags().StringVar(&config.InitOptions.GitServer.PullUsername, "git-pull-username", "", "Username for pull-only access to the git server") + initCmd.Flags().StringVar(&config.InitOptions.GitServer.PullPassword, "git-pull-password", "", "Password for the pull-only user to access the git server") + + // Flags for using an external registry + initCmd.Flags().StringVar(&config.InitOptions.RegistryInfo.Address, "registry-url", "", "External registry url address to use for this Zarf cluster") + initCmd.Flags().IntVar(&config.InitOptions.RegistryInfo.NodePort, "nodeport", 0, "Nodeport to access a registry internal to the k8s cluster. Between [30000-32767]") + initCmd.Flags().StringVar(&config.InitOptions.RegistryInfo.PushUsername, "registry-push-username", config.ZarfRegistryPushUser, "Username to access to the registry Zarf is configured to use") + initCmd.Flags().StringVar(&config.InitOptions.RegistryInfo.PushPassword, "registry-push-password", "", "Password for the push-user to connect to the registry") + initCmd.Flags().StringVar(&config.InitOptions.RegistryInfo.PullUsername, "registry-pull-username", "", "Username for pull-only access to the registry") + initCmd.Flags().StringVar(&config.InitOptions.RegistryInfo.PullPassword, "registry-pull-password", "", "Password for the pull-only user to access the registry") + initCmd.Flags().StringVar(&config.InitOptions.RegistryInfo.Secret, "registry-secret", "", "Registry secret value") + + initCmd.Flags().SortFlags = true } diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index 079a6461c0..09e6c9c431 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -37,7 +37,7 @@ var prepareTransformGitLinks = &cobra.Command{ // Perform git url transformation via regex text := string(content) - processedText := git.MutateGitUrlsInText(host, text) + processedText := git.MutateGitUrlsInText(host, text, config.InitOptions.GitServer.PushUsername) // Ask the user before this destructive action confirm := false @@ -103,4 +103,6 @@ func init() { prepareFindImages.Flags().StringVarP(&repoHelmChartPath, "repo-chart-path", "p", "", `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"`) prepareFindImages.Flags().StringVar(&config.CommonOptions.TempDirectory, "tmpdir", "", "Specify the temporary directory to use for intermediate files") prepareFindImages.Flags().StringToStringVar(&config.CommonOptions.SetVariables, "set", map[string]string{}, "Specify package variables to set on the command line (KEY=value)") + + prepareTransformGitLinks.Flags().StringVar(&config.InitOptions.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, "User or organization name for the git account that the repos are created under.") } diff --git a/src/cmd/tools.go b/src/cmd/tools.go index 5dbd611be7..63542516f2 100644 --- a/src/cmd/tools.go +++ b/src/cmd/tools.go @@ -62,8 +62,9 @@ var registryCmd = &cobra.Command{ } var readCredsCmd = &cobra.Command{ - Use: "get-admin-password", - Short: "Returns the Zarf admin password for gitea read from the zarf-state secret in the zarf namespace", + Use: "get-git-password", + Short: "Returns the push user's password for the Git server", + Long: "Reads the password for a user with push access to the configured Git server from the zarf-state secret in the zarf namespace", Run: func(cmd *cobra.Command, args []string) { state, err := k8s.LoadZarfState() if err != nil { @@ -78,7 +79,8 @@ var readCredsCmd = &cobra.Command{ // Continue loading state data if it is valid config.InitState(state) - fmt.Println(config.GetSecret(config.StateGitPush)) + message.Note("Git Server Push Password: ") + fmt.Println(state.GitServer.PushPassword) }, } diff --git a/src/config/config.go b/src/config/config.go index a5cbb29d1d..a3e95bbed0 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -25,13 +25,15 @@ const ( PackagePrefix = "zarf-package" // ZarfMaxChartNameLength limits helm chart name size to account for K8s/helm limits and zarf prefix - ZarfMaxChartNameLength = 40 - ZarfGitPushUser = "zarf-git-user" - ZarfGitReadUser = "zarf-git-read-user" - ZarfRegistryPushUser = "zarf-push" - ZarfRegistryPullUser = "zarf-pull" - ZarfImagePullSecretName = "private-registry" - ZarfGitServerSecretName = "private-git-server" + ZarfMaxChartNameLength = 40 + ZarfGitPushUser = "zarf-git-user" + ZarfGitReadUser = "zarf-git-read-user" + ZarfRegistryPushUser = "zarf-push" + ZarfRegistryPullUser = "zarf-pull" + ZarfImagePullSecretName = "private-registry" + ZarfGitServerSecretName = "private-git-server" + ZarfGeneratedPasswordLen = 24 + ZarfGeneratedSecretLen = 48 ZarfAgentHost = "agent-hook.zarf.svc" @@ -43,7 +45,13 @@ const ( ZarfCleanupScriptsPath = "/opt/zarf" ZarfDefaultImageCachePath = ".zarf-image-cache" - ZarfYAML = "zarf.yaml" + ZarfYAML = "zarf.yaml" + ZarfSBOMDir = "zarf-sbom" + + ZarfInClusterContainerRegistryURL = "http://zarf-registry-http.zarf.svc.cluster.local:5000" + ZarfInClusterContainerRegistryNodePort = 31999 + + ZarfInClusterGitServiceURL = "http://zarf-gitea-http.zarf.svc.cluster.local:3000" ) var ( @@ -59,8 +67,13 @@ var ( // DeployOptions tracks user-defined values for the active deployment DeployOptions types.ZarfDeployOptions + // InitOptions tracks user-defined values for the active Zarf initialization. + InitOptions types.ZarfInitOptions + + // CliArch is the computer architecture of the device executing the CLI commands CliArch string + // ZarfSeedPort is the NodePort Zarf uses for the 'seed registry' ZarfSeedPort string // Private vars @@ -189,7 +202,6 @@ func GetValidPackageExtensions() [3]string { func InitState(tmpState types.ZarfState) { message.Debugf("config.InitState()") state = tmpState - initSecrets() } func GetState() types.ZarfState { @@ -197,7 +209,12 @@ func GetState() types.ZarfState { } func GetRegistry() string { - return fmt.Sprintf("%s:%s", IPV4Localhost, state.NodePort) + // If a node port is populated, then we are using a registry internal to the cluster. Ignore the provided address and use localhost + if state.RegistryInfo.NodePort >= 30000 { + return fmt.Sprintf("%s:%d", IPV4Localhost, state.RegistryInfo.NodePort) + } + + return state.RegistryInfo.Address } // LoadConfig loads the config from the given path and removes @@ -224,6 +241,16 @@ func GetActiveConfig() types.ZarfPackage { return active } +// GetGitServerInfo returns the GitServerInfo for the git server Zarf is configured to use from the state +func GetGitServerInfo() types.GitServerInfo { + return state.GitServer +} + +// GetContainerRegistryInfo returns the ContainerRegistryInfo for the docker registry Zarf is configured to use from the state +func GetContainerRegistryInfo() types.RegistryInfo { + return state.RegistryInfo +} + // BuildConfig adds build information and writes the config to the given path func BuildConfig(path string) error { message.Debugf("config.BuildConfig(%s)", path) diff --git a/src/config/secret.go b/src/config/secret.go deleted file mode 100644 index 7d1a4f8a75..0000000000 --- a/src/config/secret.go +++ /dev/null @@ -1,74 +0,0 @@ -package config - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - - "github.com/defenseunicorns/zarf/src/internal/message" -) - -type SecretSelector = string - -type SecretMap struct { - length int - computed string - valid bool -} - -const ( - StateRegistryPush SecretSelector = "registry-push" - StateRegistryPull SecretSelector = "registry-pull" - StateRegistrySecret SecretSelector = "registry-secret" - StateGitPush SecretSelector = "git-push" - StateGitPull SecretSelector = "git-pull" - StateLogging SecretSelector = "logging" -) - -var selectors = map[SecretSelector]SecretMap{ - StateRegistryPush: {length: 48}, - StateRegistryPull: {length: 48}, - StateRegistrySecret: {length: 48}, - StateGitPush: {length: 24}, - StateGitPull: {length: 24}, - StateLogging: {length: 24}, -} - -func GetSecret(selector SecretSelector) string { - message.Debugf("config.GetSecret(%s)", selector) - if match, ok := selectors[selector]; ok { - return match.computed - } - return "" -} - -func initSecrets() { - message.Debug("config.initSecrets()") - for filter, selector := range selectors { - output, err := loadSecret(filter, selector.length) - if err != nil { - message.Debug(err) - } else { - selector.valid = true - selector.computed = output - selectors[filter] = selector - } - } -} - -func loadSecret(filter SecretSelector, length int) (string, error) { - message.Debugf("config.loadSecret(%s, %d)", filter, length) - if state.Secret == "" { - return "", fmt.Errorf("invalid root secret in the ZarfState") - } - hash := sha256.New() - text := fmt.Sprintf("%s:%s", filter, state.Secret) - hash.Write([]byte(text)) - output := hex.EncodeToString(hash.Sum(nil))[:length] - - if output != "" { - return output, nil - } else { - return "", fmt.Errorf("unable to generate secret for %s", filter) - } -} diff --git a/src/internal/agent/hooks/flux.go b/src/internal/agent/hooks/flux.go index 82d803ee99..754b7eae98 100644 --- a/src/internal/agent/hooks/flux.go +++ b/src/internal/agent/hooks/flux.go @@ -8,9 +8,12 @@ import ( "github.com/defenseunicorns/zarf/src/internal/agent/operations" "github.com/defenseunicorns/zarf/src/internal/git" "github.com/defenseunicorns/zarf/src/internal/message" + "github.com/defenseunicorns/zarf/src/internal/utils" v1 "k8s.io/api/admission/v1" ) +const zarfStatePath = "/etc/zarf-state/state" + type SecretRef struct { Name string `json:"name"` } @@ -26,35 +29,69 @@ type GenericGitRepo struct { func NewGitRepositoryMutationHook() operations.Hook { message.Debug("hooks.NewGitRepositoryMutationHook()") return operations.Hook{ - Create: mutateGitRepository, - Update: mutateGitRepository, + Create: mutateGitRepo, + Update: mutateGitRepo, } } -func mutateGitRepository(r *v1.AdmissionRequest) (*operations.Result, error) { +// mutateGitRepoCreate mutates the git repository url to point to the repository URL defined in the zarfState. +func mutateGitRepo(r *v1.AdmissionRequest) (*operations.Result, error) { var patches []operations.PatchOperation + // Form the gitServerURL from the state + zarfState, err := getStateFromAgentPod(zarfStatePath) + if err != nil { + return nil, fmt.Errorf("failed to load zarf state from file: %w", err) + } + gitServerURL := zarfState.GitServer.Address + message.Debugf("Using the gitServerURL of (%s) to mutate the flux repository", gitServerURL) + // parse to simple struct to read the git url gitRepo := &GenericGitRepo{} if err := json.Unmarshal(r.Object.Raw, &gitRepo); err != nil { - return nil, fmt.Errorf("failed to unmarshal manifest: %v", err) + return nil, fmt.Errorf("failed to unmarshal manifest: %w", err) + } + gitURL := gitRepo.Spec.URL + + // Check if this is an update operation and the hostname is different from what we have in the state + // NOTE: We mutate on updates IF AND ONLY IF the hostname in the request is different than the hostname in the zarfState + // NOTE: We are checking if the hostname is different before because we do not want to potentially mutate a URL that has already been mutated. + urlMatches := false + if r.Operation == v1.Update { + urlMatches, err = utils.DoesHostnamesMatch(gitServerURL, gitRepo.Spec.URL) + if err != nil { + return nil, fmt.Errorf("failed to complete hostname matching: %w", err) + } } - message.Info(gitRepo.Spec.URL) + // Mutate the git URL if necessary + if r.Operation == v1.Create || (r.Operation == v1.Update && !urlMatches) { + // Mutate the git URL so that the hostname matches the hostname in the Zarf state + gitURL = git.MutateGitUrlsInText(gitServerURL, gitURL, zarfState.GitServer.PushUsername) + message.Debugf("original git URL of (%s) got mutated to (%s)", gitRepo.Spec.URL, gitURL) + } + + // Patch updates of the repo spec + patches = populatePatchOperations(gitURL, gitRepo.Spec.SecretRef.Name) + + return &operations.Result{ + Allowed: true, + PatchOps: patches, + }, nil +} - replacedURL := git.MutateGitUrlsInText("http://zarf-gitea-http.zarf.svc.cluster.local:3000", gitRepo.Spec.URL) - patches = append(patches, operations.ReplacePatchOperation("/spec/url", replacedURL)) +// Patch updates of the repo spec. +func populatePatchOperations(repoURL string, secretName string) []operations.PatchOperation { + var patches []operations.PatchOperation + patches = append(patches, operations.ReplacePatchOperation("/spec/url", repoURL)) // If a prior secret exists, replace it - if gitRepo.Spec.SecretRef.Name != "" { + if secretName != "" { patches = append(patches, operations.ReplacePatchOperation("/spec/secretRef/name", config.ZarfGitServerSecretName)) } else { // Otherwise, add the new secret patches = append(patches, operations.AddPatchOperation("/spec/secretRef", SecretRef{Name: config.ZarfGitServerSecretName})) } - return &operations.Result{ - Allowed: true, - PatchOps: patches, - }, nil + return patches } diff --git a/src/internal/agent/hooks/pods.go b/src/internal/agent/hooks/pods.go index 588902880f..628af5ec5c 100644 --- a/src/internal/agent/hooks/pods.go +++ b/src/internal/agent/hooks/pods.go @@ -3,11 +3,13 @@ package hooks import ( "encoding/json" "fmt" + "os" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/agent/operations" "github.com/defenseunicorns/zarf/src/internal/message" "github.com/defenseunicorns/zarf/src/internal/utils" + "github.com/defenseunicorns/zarf/src/types" v1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" @@ -24,7 +26,6 @@ func NewPodMutationHook() operations.Hook { func parsePod(object []byte) (*corev1.Pod, error) { message.Debugf("pods.parsePod(%s)", string(object)) - var pod corev1.Pod if err := json.Unmarshal(object, &pod); err != nil { return nil, err @@ -54,24 +55,44 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { zarfSecret := []corev1.LocalObjectReference{{Name: config.ZarfImagePullSecretName}} patchOperations = append(patchOperations, operations.ReplacePatchOperation("/spec/imagePullSecrets", zarfSecret)) + zarfState, err := getStateFromAgentPod(zarfStatePath) + if err != nil { + message.Debugf("Unable to load the ZarfState file so that the Agent can mutate pods: %#v", err) + return nil, err + } + config.InitState(zarfState) + containerRegistryURL := config.GetRegistry() + // update the image host for each init container for idx, container := range pod.Spec.InitContainers { path := fmt.Sprintf("/spec/initContainers/%d/image", idx) - replacement := utils.SwapHost(container.Image, "127.0.0.1:31999") + replacement, err := utils.SwapHost(container.Image, containerRegistryURL) + if err != nil { + message.Warnf("Unable to swap the host for (%s)", container.Image) + continue // Continue, because we might as well attempt to mutate the other containers for this pod + } patchOperations = append(patchOperations, operations.ReplacePatchOperation(path, replacement)) } // update the image host for each ephemeral container for idx, container := range pod.Spec.EphemeralContainers { path := fmt.Sprintf("/spec/ephemeralContainers/%d/image", idx) - replacement := utils.SwapHost(container.Image, "127.0.0.1:31999") + replacement, err := utils.SwapHost(container.Image, containerRegistryURL) + if err != nil { + message.Warnf("Unable to swap the host for (%s)", container.Image) + continue // Continue, because we might as well attempt to mutate the other containers for this pod + } patchOperations = append(patchOperations, operations.ReplacePatchOperation(path, replacement)) } // update the image host for each normal container for idx, container := range pod.Spec.Containers { path := fmt.Sprintf("/spec/containers/%d/image", idx) - replacement := utils.SwapHost(container.Image, "127.0.0.1:31999") + replacement, err := utils.SwapHost(container.Image, containerRegistryURL) + if err != nil { + message.Warnf("Unable to swap the host for (%s)", container.Image) + continue // Continue, because we might as well attempt to mutate the other containers for this pod + } patchOperations = append(patchOperations, operations.ReplacePatchOperation(path, replacement)) } @@ -83,3 +104,26 @@ func mutatePod(r *v1.AdmissionRequest) (*operations.Result, error) { PatchOps: patchOperations, }, nil } + +// Reads the state json file that was mounted into the agent pods +func getStateFromAgentPod(zarfStatePath string) (types.ZarfState, error) { + zarfState := types.ZarfState{} + + // Read the state file + stateFile, err := os.ReadFile(zarfStatePath) + if err != nil { + message.Warnf("Unable to read the zarfState file within the zarf-agent pod.") + return zarfState, err + } + + // Unmarshal the json file into a Go struct + err = json.Unmarshal(stateFile, &zarfState) + if err != nil { + message.Warnf("Unable to unmarshal the zarfState file into a useable object.") + return zarfState, err + } + + message.Debugf("ZarfState from file = %#v", zarfState) + + return zarfState, err +} diff --git a/src/internal/git/pull.go b/src/internal/git/pull.go index bf2c9c1dc2..73af7fea35 100644 --- a/src/internal/git/pull.go +++ b/src/internal/git/pull.go @@ -24,10 +24,16 @@ func DownloadRepoToTemp(gitUrl string, spinner *message.Spinner) string { return path } -func Pull(gitUrl, targetFolder string, spinner *message.Spinner) string { - path := targetFolder + "/" + transformURLtoRepoName(gitUrl) +func Pull(gitUrl, targetFolder string, spinner *message.Spinner) (string, error) { + repoName, err := transformURLtoRepoName(gitUrl) + if err != nil { + message.Errorf(err, "unable to pull the git repo at %s", gitUrl) + return "", err + } + + path := targetFolder + "/" + repoName pull(gitUrl, path, spinner) - return path + return path, nil } func pull(gitUrl, targetFolder string, spinner *message.Spinner) { diff --git a/src/internal/git/push.go b/src/internal/git/push.go index 89b4d73459..b4aeedee44 100644 --- a/src/internal/git/push.go +++ b/src/internal/git/push.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "path/filepath" - "strings" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/k8s" @@ -20,12 +19,17 @@ const offlineRemoteName = "offline-downstream" const onlineRemoteRefPrefix = "refs/remotes/" + onlineRemoteName + "/" func PushAllDirectories(localPath string) error { - // Establish a git tunnel to send the repos - tunnel := k8s.NewZarfTunnel() - tunnel.Connect(k8s.ZarfGit, false) - defer tunnel.Close() - - tunnelUrl := fmt.Sprintf("http://%s", tunnel.Endpoint()) + gitServerInfo := config.GetGitServerInfo() + gitServerURL := gitServerInfo.Address + + // If this is a serviceURL, create a port-forward tunnel to that resource + if tunnel, err := k8s.NewTunnelFromServiceURL(gitServerURL); err != nil { + message.Debug(err) + } else { + tunnel.Connect("", false) + defer tunnel.Close() + gitServerURL = fmt.Sprintf("http://%s", tunnel.Endpoint()) + } paths, err := utils.ListDirectories(localPath) if err != nil { @@ -39,19 +43,38 @@ func PushAllDirectories(localPath string) error { for _, path := range paths { basename := filepath.Base(path) spinner.Updatef("Pushing git repo %s", basename) - if err := push(path, tunnelUrl, spinner); err != nil { + + repo, err := prepRepoForPush(path, gitServerURL, gitServerInfo.PushUsername) + if err != nil { + message.Warnf("error when preping the repo for push.. %v", err) + return err + } + + if err := push(repo, path, spinner); err != nil { spinner.Warnf("Unable to push the git repo %s", basename) return err } // Add the read-only user to this repo - repoPathSplit := strings.Split(path, "/") - repoNameWithGitTag := repoPathSplit[len(repoPathSplit)-1] - repoName := strings.Split(repoNameWithGitTag, ".git")[0] - err = addReadOnlyUserToRepo(tunnelUrl, repoName) - if err != nil { - message.Warnf("Unable to add the read-only user to the repo: %s\n", repoName) - return err + if gitServerInfo.InternalServer { + // Get the upstream URL + remote, err := repo.Remote(onlineRemoteName) + if err != nil { + message.Warn("unable to get the information needed to add the read-only user to the repo") + return err + } + remoteUrl := remote.Config().URLs[0] + repoName, err := transformURLtoRepoName(remoteUrl) + if err != nil { + message.Warnf("Unable to add the read-only user to the repo: %s\n", repoName) + return err + } + + err = addReadOnlyUserToRepo(gitServerURL, repoName) + if err != nil { + message.Warnf("Unable to add the read-only user to the repo: %s\n", repoName) + return err + } } } @@ -59,35 +82,40 @@ func PushAllDirectories(localPath string) error { return nil } -func push(localPath, tunnelUrl string, spinner *message.Spinner) error { - +func prepRepoForPush(localPath, tunnelUrl, username string) (*git.Repository, error) { // Open the given repo repo, err := git.PlainOpen(localPath) if err != nil { - return fmt.Errorf("not a valid git repo or unable to open: %w", err) + return nil, fmt.Errorf("not a valid git repo or unable to open: %w", err) } // Get the upstream URL remote, err := repo.Remote(onlineRemoteName) if err != nil { - return fmt.Errorf("unable to find the git remote: %w", err) - + return nil, fmt.Errorf("unable to find the git remote: %w", err) } + remoteUrl := remote.Config().URLs[0] - targetUrl := transformURL(tunnelUrl, remoteUrl) + targetUrl, err := transformURL(tunnelUrl, remoteUrl, username) + if err != nil { + return nil, fmt.Errorf("unable to transform the git url: %w", err) + } _, err = repo.CreateRemote(&goConfig.RemoteConfig{ Name: offlineRemoteName, URLs: []string{targetUrl}, }) - if err != nil { - return fmt.Errorf("failed to create offline remote: %w", err) + return nil, fmt.Errorf("failed to create offline remote: %w", err) } + return repo, nil +} + +func push(repo *git.Repository, localPath string, spinner *message.Spinner) error { gitCred := http.BasicAuth{ - Username: config.ZarfGitPushUser, - Password: config.GetSecret(config.StateGitPush), + Username: config.GetState().GitServer.PushUsername, + Password: config.GetState().GitServer.PushPassword, } // Since we are pushing HEAD:refs/heads/master on deployment, leaving @@ -109,11 +137,16 @@ func push(localPath, tunnelUrl string, spinner *message.Spinner) error { }, } + // Attempt the fetch, if it fails, log a warning and continue trying to push (might as well try..) err = repo.Fetch(fetchOptions) if errors.Is(err, transport.ErrRepositoryNotFound) { - message.Debugf("Repo not yet available offline, skipping fetch") + message.Debugf("Repo not yet available offline, skipping fetch...") + } else if errors.Is(err, git.ErrForceNeeded) { + message.Debugf("Repo fetch requires force, skipping fetch...") + } else if errors.Is(err, git.NoErrAlreadyUpToDate) { + message.Debugf("Repo already up-to-date, skipping fetch...") } else if err != nil { - return fmt.Errorf("unable to fetch remote cleanly prior to push: %w", err) + message.Warnf("unable to fetch remote cleanly prior to push: %#v", err) } // Push all heads and tags to the offline remote diff --git a/src/internal/git/utils.go b/src/internal/git/utils.go index c15320bd9d..7116280e40 100644 --- a/src/internal/git/utils.go +++ b/src/internal/git/utils.go @@ -3,6 +3,8 @@ package git import ( "bufio" "bytes" + "crypto/sha1" + "encoding/hex" "encoding/json" "fmt" "io" @@ -26,28 +28,50 @@ type Credential struct { Auth http.BasicAuth } -func MutateGitUrlsInText(host string, text string) string { +// MutateGitURlsInText Changes the giturl hostname to use the repository Zarf is configured to use +func MutateGitUrlsInText(host string, text string, gitUser string) string { extractPathRegex := regexp.MustCompilePOSIX(`https?://[^/]+/(.*\.git)`) output := extractPathRegex.ReplaceAllStringFunc(text, func(match string) string { - if strings.Contains(match, "/"+config.ZarfGitPushUser+"/") { - message.Warnf("%s seems to have been previously patched.", match) - return match + output, err := transformURL(host, match, gitUser) + if err != nil { + message.Warnf("Unable to transform the git url, using the original url we have: %s", match) + output = match } - return transformURL(host, match) + return output }) return output } -func transformURLtoRepoName(url string) string { - replaceRegex := regexp.MustCompile(`(https?://|[^\w\-.])+`) - return "mirror" + replaceRegex.ReplaceAllString(url, "__") +func transformURLtoRepoName(url string) (string, error) { + // For further explanation: https://regex101.com/library/UfILls and https://regex101.com/rary/UfILls + findRegex := regexp.MustCompile(`\/([\w\-]+)(.git)?(@([\w\-\.]+))?$`) + substrings := findRegex.FindStringSubmatch(url) + if len(substrings) == 0 { + // the first element in the return substrings is + return "", fmt.Errorf("unable to get extract the repoName from the url %s", url) + } + + // NOTE: The first element in the returned substrings is the combination of all the rest of the substrings.... + // So just skip the first element so we can get a hash without the version tag + repoName := substrings[1] + + // Add sha1 hash of the repoName to the end of the repo + hasher := sha1.New() + _, _ = io.WriteString(hasher, url) + sha1Hash := hex.EncodeToString(hasher.Sum(nil)) + newRepoName := repoName + "-" + sha1Hash + + return newRepoName, nil } -func transformURL(baseUrl string, url string) string { - replaced := transformURLtoRepoName(url) - output := baseUrl + "/" + config.ZarfGitPushUser + "/" + replaced +func transformURL(baseUrl string, url string, username string) (string, error) { + replaced, err := transformURLtoRepoName(url) + if err != nil { + return "", err + } + output := baseUrl + "/" + username + "/" + replaced message.Debugf("Rewrite git URL: %s -> %s", url, output) - return output + return output, nil } func credentialFilePath() string { @@ -248,11 +272,12 @@ func CreateReadOnlyUser() error { defer tunnel.Close() tunnelUrl := tunnel.Endpoint() + zarfState := config.GetState() // Create json representation of the create-user request body createUserBody := map[string]interface{}{ - "username": config.ZarfGitReadUser, - "password": config.GetSecret(config.StateGitPull), + "username": zarfState.GitServer.PullUsername, + "password": zarfState.GitServer.PullPassword, "email": "zarf-reader@localhost.local", "must_change_password": false, } @@ -264,7 +289,7 @@ func CreateReadOnlyUser() error { // Send API request to create the user createUserEndpoint := fmt.Sprintf("http://%s/api/v1/admin/users", tunnelUrl) createUserRequest, _ := netHttp.NewRequest("POST", createUserEndpoint, bytes.NewBuffer(createUserData)) - out, err := DoHttpThings(createUserRequest, config.ZarfGitPushUser, config.GetSecret(config.StateGitPush)) + out, err := DoHttpThings(createUserRequest, zarfState.GitServer.PushUsername, zarfState.GitServer.PushPassword) message.Debugf("POST %s:\n%s", createUserEndpoint, string(out)) if err != nil { return err @@ -272,14 +297,14 @@ func CreateReadOnlyUser() error { // Make sure the user can't create their own repos or orgs updateUserBody := map[string]interface{}{ - "login_name": config.ZarfGitReadUser, + "login_name": zarfState.GitServer.PushUsername, "max_repo_creation": 0, "allow_create_organization": false, } updateUserData, _ := json.Marshal(updateUserBody) - updateUserEndpoint := fmt.Sprintf("http://%s/api/v1/admin/users/%s", tunnelUrl, config.ZarfGitReadUser) + updateUserEndpoint := fmt.Sprintf("http://%s/api/v1/admin/users/%s", tunnelUrl, zarfState.GitServer.PullUsername) updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) - out, err = DoHttpThings(updateUserRequest, config.ZarfGitPushUser, config.GetSecret(config.StateGitPush)) + out, err = DoHttpThings(updateUserRequest, zarfState.GitServer.PushUsername, zarfState.GitServer.PushPassword) message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) return err } @@ -295,9 +320,9 @@ func addReadOnlyUserToRepo(tunnelUrl, repo string) error { } // Send API request to add a user as a read-only collaborator to a repo - addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelUrl, config.ZarfGitPushUser, repo, config.ZarfGitReadUser) + addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelUrl, config.GetState().GitServer.PushUsername, repo, config.GetState().GitServer.PullUsername) addColabRequest, _ := netHttp.NewRequest("PUT", addColabEndpoint, bytes.NewBuffer(addColabData)) - out, err := DoHttpThings(addColabRequest, config.ZarfGitPushUser, config.GetSecret(config.StateGitPush)) + out, err := DoHttpThings(addColabRequest, config.GetState().GitServer.PushUsername, config.GetState().GitServer.PushPassword) message.Debugf("PUT %s:\n%s", addColabEndpoint, string(out)) return err } diff --git a/src/internal/helm/post-render.go b/src/internal/helm/post-render.go index 2e618fc51a..39d2118db0 100644 --- a/src/internal/helm/post-render.go +++ b/src/internal/helm/post-render.go @@ -171,8 +171,8 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { // Generate the git server secret gitServerSecret := k8s.GenerateSecret(name, config.ZarfGitServerSecretName, corev1.SecretTypeOpaque) gitServerSecret.StringData = map[string]string{ - "username": config.ZarfGitReadUser, - "password": config.GetSecret(config.StateGitPull), + "username": config.GetGitServerInfo().PullUsername, + "password": config.GetGitServerInfo().PullPassword, } // Update the git server secret diff --git a/src/internal/images/push.go b/src/internal/images/push.go index a080bdd517..286016e8b7 100644 --- a/src/internal/images/push.go +++ b/src/internal/images/push.go @@ -8,21 +8,37 @@ import ( "github.com/google/go-containerregistry/pkg/crane" ) -func PushToZarfRegistry(imageTarballPath string, buildImageList []string) error { +// PushToZarfRegistry pushes a provided image into the configured Zarf registry +// This function will optionally shorten the image name while appending a sha1sum of the original image name +func PushToZarfRegistry(imageTarballPath string, buildImageList []string, addShasumToImg bool) error { message.Debugf("images.PushToZarfRegistry(%s, %s)", imageTarballPath, buildImageList) - // Establish a registry tunnel to send the images to the zarf registry - tunnel := k8s.NewZarfTunnel() - tunnel.Connect(k8s.ZarfRegistry, false) - defer tunnel.Close() + registryUrl := "" + if config.GetContainerRegistryInfo().InternalRegistry { + // Establish a registry tunnel to send the images to the zarf registry + tunnel := k8s.NewZarfTunnel() + tunnel.Connect(k8s.ZarfRegistry, false) + defer tunnel.Close() - tunnelUrl := tunnel.Endpoint() + registryUrl = tunnel.Endpoint() + } else { + registryUrl = config.GetContainerRegistryInfo().Address + + // If this is a serviceURL, create a port-forward tunnel to that resource + if tunnel, err := k8s.NewTunnelFromServiceURL(registryUrl); err != nil { + message.Debug(err) + } else { + tunnel.Connect("", false) + defer tunnel.Close() + registryUrl = tunnel.Endpoint() + } + } spinner := message.NewProgressSpinner("Storing images in the zarf registry") defer spinner.Stop() - pushOptions := config.GetCraneAuthOption(config.ZarfRegistryPushUser, config.GetSecret(config.StateRegistryPush)) - message.Debug(pushOptions) + pushOptions := config.GetCraneAuthOption(config.GetContainerRegistryInfo().PushUsername, config.GetContainerRegistryInfo().PushPassword) + message.Debugf("crane pushOptions = %#v", pushOptions) for _, src := range buildImageList { spinner.Updatef("Updating image %s", src) @@ -30,8 +46,16 @@ func PushToZarfRegistry(imageTarballPath string, buildImageList []string) error if err != nil { return err } + offlineName := "" + if addShasumToImg { + offlineName, err = utils.SwapHost(src, registryUrl) + } else { + offlineName, err = utils.SwapHostWithoutSha(src, registryUrl) + } + if err != nil { + return err + } - offlineName := utils.SwapHost(src, tunnelUrl) if err = crane.Push(img, offlineName, pushOptions); err != nil { return err } diff --git a/src/internal/k8s/secrets.go b/src/internal/k8s/secrets.go index af62397552..9f6d308f73 100644 --- a/src/internal/k8s/secrets.go +++ b/src/internal/k8s/secrets.go @@ -71,12 +71,18 @@ func GenerateRegistryPullCreds(namespace, name string) *corev1.Secret { secretDockerConfig := GenerateSecret(namespace, name, corev1.SecretTypeDockerConfigJson) - // Auth field must be username:password and base64 encoded - credential := config.GetSecret(config.StateRegistryPull) + // Get the registry credentials from the ZarfState secret + zarfState, err := LoadZarfState() + if err != nil { + message.Fatalf(err, "Unable to load the Zarf state to get the registry credentials") + } + credential := zarfState.RegistryInfo.PullPassword if credential == "" { message.Fatalf(nil, "Generate pull cred failed") } - fieldValue := config.ZarfRegistryPullUser + ":" + credential + + // Auth field must be username:password and base64 encoded + fieldValue := zarfState.RegistryInfo.PullUsername + ":" + credential authEncodedValue := base64.StdEncoding.EncodeToString([]byte(fieldValue)) registry := config.GetRegistry() diff --git a/src/internal/k8s/tunnel.go b/src/internal/k8s/tunnel.go index ca77bf944f..a87104ff43 100644 --- a/src/internal/k8s/tunnel.go +++ b/src/internal/k8s/tunnel.go @@ -3,12 +3,15 @@ package k8s // Forked from https://github.com/gruntwork-io/terratest/blob/v0.38.8/modules/k8s/tunnel.go import ( + "errors" "fmt" "io" "net" "net/http" + "net/url" "os" "os/signal" + "regexp" "runtime" "strconv" "strings" @@ -87,6 +90,37 @@ func PrintConnectTable() error { return nil } +// NewTunnelFromServiceURL takes a serviceURL and parses it to create a tunnel to the cluster. The string is expected to follow the following format: +// Example serviceURL: http://{SERVICE_NAME}.{NAMESPACE}.svc.cluster.local:{PORT} +func NewTunnelFromServiceURL(serviceURL string) (*Tunnel, error) { + parsedURL, err := url.Parse(serviceURL) + if err != nil { + return nil, err + } + + // Get the remote port from the serviceURL + remotePort, err := strconv.Atoi(parsedURL.Port()) + if err != nil { + return nil, err + } + + // Match hostname against local cluster service format + // See https://regex101.com/r/OWVfAO/1 + pattern := regexp.MustCompile(`^(?P[^\.]+)\.(?P[^\.]+)\.svc\.cluster\.local$`) + matches := pattern.FindStringSubmatch(parsedURL.Hostname()) + + // If incomplete match, return an error + if len(matches) != 3 { + return nil, errors.New("url does not match service url format http://{SERVICE_NAME}.{NAMESPACE}.svc.cluster.local:{PORT}") + } + + // Use the matched values to create a new tunnel + name := matches[pattern.SubexpIndex("name")] + namespace := matches[pattern.SubexpIndex("namespace")] + + return NewTunnel(namespace, SvcResource, name, 0, remotePort), nil +} + // NewTunnel will create a new Tunnel struct // Note that if you use 0 for the local port, an open port on the host system // will be selected automatically, and the Tunnel struct will be updated with the selected port. @@ -123,6 +157,7 @@ func (tunnel *Tunnel) Connect(target string, blocking bool) { case ZarfRegistry: tunnel.resourceName = "zarf-docker-registry" tunnel.remotePort = 5000 + tunnel.urlSuffix = `/v2/_catalog` case ZarfLogging: tunnel.resourceName = "zarf-loki-stack-grafana" diff --git a/src/internal/message/message.go b/src/internal/message/message.go index ecd2579b8a..48778c56eb 100644 --- a/src/internal/message/message.go +++ b/src/internal/message/message.go @@ -175,6 +175,12 @@ func Note(text string) { pterm.FgYellow.Println(message) } +func Notef(text string, a ...any) { + pterm.Println() + message := paragraph(text, a...) + pterm.FgYellow.Println(message) +} + func HeaderInfof(format string, a ...any) { message := fmt.Sprintf(format, a...) // Ensure the text is consistent for the header width diff --git a/src/internal/packager/components.go b/src/internal/packager/components.go index 4e786eb9e0..7206f85386 100644 --- a/src/internal/packager/components.go +++ b/src/internal/packager/components.go @@ -64,7 +64,7 @@ func getValidComponents(allComponents []types.ZarfComponent, requestedComponentN if requested { // Mark deployment as appliance mode if this is an init config and the k3s component is enabled if component.Name == k8s.DistroIsK3s && config.IsZarfInitConfig() { - config.DeployOptions.ApplianceMode = true + config.InitOptions.ApplianceMode = true } // Add the component to the list of valid components validComponentsList = append(validComponentsList, component) diff --git a/src/internal/packager/create.go b/src/internal/packager/create.go index b49220e24a..90ec1b0882 100644 --- a/src/internal/packager/create.go +++ b/src/internal/packager/create.go @@ -210,7 +210,10 @@ func addComponent(tempPath tempPaths, component types.ZarfComponent) { defer spinner.Success() for _, url := range component.Repos { // Pull all the references if there is no `@` in the string - git.Pull(url, componentPath.repos, spinner) + _, err := git.Pull(url, componentPath.repos, spinner) + if err != nil { + message.Fatalf(err, fmt.Sprintf("Unable to pull the repo with the url of (%s}", url)) + } } } diff --git a/src/internal/packager/deploy.go b/src/internal/packager/deploy.go index 14bb6e0b18..c4b4d143fa 100644 --- a/src/internal/packager/deploy.go +++ b/src/internal/packager/deploy.go @@ -5,6 +5,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "strconv" "strings" @@ -30,6 +31,7 @@ import ( var valueTemplate template.Values var connectStrings = make(types.ConnectStrings) +// Deploy attempts to deploy a Zarf package that is define within the global DeployOptions struct func Deploy() { message.Debug("packager.Deploy()") @@ -65,26 +67,12 @@ func Deploy() { spinner.Success() - sbomViewFiles, _ := filepath.Glob(tempPath.sboms + "/sbom-viewer-*") // If SBOM files exist, temporary place them in the deploy directory - if len(sbomViewFiles) > 0 { - sbomDir := "zarf-sbom" - // Cleanup any failed prior removals - _ = os.RemoveAll(sbomDir) - // Create the directory again - utils.CreateDirectory(sbomDir, 0755) - for _, file := range sbomViewFiles { - // Our file copy lib explodes on these files for some reason... - data, err := ioutil.ReadFile(file) - if err != nil { - message.Fatalf(err, "Unable to read the sbom-viewer file %s", file) - } - dst := filepath.Join(sbomDir, filepath.Base(file)) - err = ioutil.WriteFile(dst, data, 0644) - if err != nil { - message.Fatalf(err, "Unable to write the sbom-viewer file %s", dst) - } - } + sbomViewFiles, _ := filepath.Glob(tempPath.sboms + "/sbom-viewer-*") + err = writeSBOMFiles(sbomViewFiles) + if err != nil { + message.Errorf(err, "Unable to process the SBOM files for this package") + // Don't stop the deployment, let the user decide if they want to continue the deployment } // Confirm the overall package deployment @@ -92,7 +80,7 @@ func Deploy() { // Don't continue unless the user says so if !confirm { - os.Exit(0) + return } // Generate a secret that describes the package that is being deployed @@ -119,251 +107,392 @@ func Deploy() { if config.DeployOptions.Components != "" { requestedComponents = strings.Split(config.DeployOptions.Components, ",") } - componentsToDeploy := getValidComponents(components, requestedComponents) - // Deploy all the components - for _, component := range componentsToDeploy { - installedCharts := deployComponents(tempPath, component) - - // Get information about what we just installed so we can save it to a secret later - installedComponent := types.DeployedComponent{Name: component.Name} - if len(installedCharts) > 0 { - installedComponent.InstalledCharts = installedCharts - } - - installedZarfPackage.DeployedComponents = append(installedZarfPackage.DeployedComponents, installedComponent) + // Get a list of all the components we are deploying and actually deploy them + componentsToDeploy := getValidComponents(components, requestedComponents) + deployedComponents, err := deployComponents(tempPath, componentsToDeploy) + if err != nil { + message.Errorf(err, "Unable to deploy all the components of this Zarf Package.") } + installedZarfPackage.DeployedComponents = deployedComponents + // Notify all the things about the successful deployment message.SuccessF("Zarf deployment complete") pterm.Println() + printTablesForDeployment(componentsToDeploy) - // If not init config, print the application connection table - if !config.IsZarfInitConfig() { - message.PrintConnectStringTable(connectStrings) - } else { - // otherwise, print the init config connection and passwords - loginTable := pterm.TableData{ - {" Application", "Username", "Password", "Connect"}, - {" Registry", config.ZarfRegistryPushUser, config.GetSecret(config.StateRegistryPush), "zarf connect registry"}, - } - for _, component := range componentsToDeploy { - // Show message if including logging stack - if component.Name == "logging" { - loginTable = append(loginTable, pterm.TableData{{" Logging", "zarf-admin", config.GetSecret(config.StateLogging), "zarf connect logging"}}...) - } - // Show message if including git-server - if component.Name == "git-server" { - loginTable = append(loginTable, pterm.TableData{ - {" Git", config.ZarfGitPushUser, config.GetSecret(config.StateGitPush), "zarf connect git"}, - {" Git (read-only)", config.ZarfGitReadUser, config.GetSecret(config.StateGitPull), "zarf connect git"}, - }...) - } - } - _ = pterm.DefaultTable.WithHasHeader().WithData(loginTable).Render() - } - - // Not all packages need k8s so we need to check if k8s is being used before saving the metadata secret + // Save deployed package information to k8s + // Note: Not all packages need k8s; check if k8s is being used before saving the secret if packageUsesK8s(config.GetActiveConfig()) { stateData, _ := json.Marshal(installedZarfPackage) deployedPackageSecret.Data = make(map[string][]byte) deployedPackageSecret.Data["data"] = stateData k8s.ReplaceSecret(deployedPackageSecret) } + + // All done + return } -func deployComponents(tempPath tempPaths, component types.ZarfComponent) []types.InstalledCharts { - message.Debugf("packager.deployComponents(%#v, %#v", tempPath, component) +// deployComponents loops through a list of ZarfComponents and deploys them +func deployComponents(tempPath tempPaths, componentsToDeploy []types.ZarfComponent) ([]types.DeployedComponent, error) { + // When pushing images, the default behavior is to add a shasum of the url to the image name + deployedComponents := []types.DeployedComponent{} - var installedCharts []types.InstalledCharts + // Deploy all the components + for _, component := range componentsToDeploy { + deployedComponent := types.DeployedComponent{Name: component.Name} + installedCharts := []types.InstalledChart{} + addShasumToImg := true + + // If this is an init-package and we are using an external registry, don't deploy the components to stand up an internal registry + // TODO: Figure out a better way to do this (I don't like how these components are still `required` according to the yaml definition) + if (config.IsZarfInitConfig() && config.InitOptions.RegistryInfo.Address != "") && + (component.Name == "zarf-seed-registry" || component.Name == "zarf-injector" || component.Name == "zarf-registry") { + message.Notef("Not deploying the component (%s) since external registry information was provided during `zarf init`", component.Name) + continue + } - // Toggles for deploy operations on 'init' - isSeedRegistry := config.IsZarfInitConfig() && component.Name == "zarf-seed-registry" + // Do somewhat custom pre-configuration for the seed and agent components + if config.IsZarfInitConfig() && component.Name == "zarf-seed-registry" && config.InitOptions.RegistryInfo.Address == "" { + // The zarf-seed-registry component is responsible for seeding the state and finding a pod to inject a registry into + seedZarfState(tempPath) + runInjectionMadness(tempPath) + } else if config.IsZarfInitConfig() && component.Name == "zarf-agent" { + // The zarf-agent cannot mutate itself, so don't change the img url + addShasumToImg = false + + // If we are using an external registry, we will need to seed the ZarfState as part of the zarf-agent component + if !config.GetContainerRegistryInfo().InternalRegistry { + seedZarfState(tempPath) + } + } + + // Actually deploy the component + installedCharts = deployComponent(tempPath, component, addShasumToImg) + + // Do cleanup for when we inject the seed registry during initialization + if config.IsZarfInitConfig() && component.Name == "zarf-seed-registry" { + err := postSeedRegistry(tempPath) + if err != nil { + message.Warnf("Unable to seed the Zarf registry") + return deployedComponents, fmt.Errorf("unable to seed the Zarf Registry: %w", err) + } + } + + // Deploy the component + deployedComponent.InstalledCharts = installedCharts + deployedComponents = append(deployedComponents, deployedComponent) + } + + return deployedComponents, nil +} + +// Deploy a Zarf Component +func deployComponent(tempPath tempPaths, component types.ZarfComponent, addShasumToImgs bool) []types.InstalledChart { + var installedCharts []types.InstalledChart + message.Debugf("packager.deployComponent(%#v, %#v", tempPath, component) // Toggles for general deploy operations componentPath := createComponentPaths(tempPath.components, component) + + // All components now require a name + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + hasImages := len(component.Images) > 0 hasCharts := len(component.Charts) > 0 hasManifests := len(component.Manifests) > 0 hasRepos := len(component.Repos) > 0 hasDataInjections := len(component.DataInjections) > 0 - // All components now require a name - message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + // Run the 'before' scripts and move files before we do anything else + runComponentScripts(component.Scripts.Before, component.Scripts) + processComponentFiles(component.Files, componentPath.files, tempPath.base) - for _, script := range component.Scripts.Before { - loopScriptUntilSuccess(script, component.Scripts) + // Generate a value template + valueTemplate = template.Generate() + if !valueTemplate.Ready() && (hasImages || hasCharts || hasManifests || hasRepos) { + valueTemplate = getUpdatedValueTemplate(component) } - if len(component.Files) > 0 { - spinner := message.NewProgressSpinner("Copying %d files", len(component.Files)) - defer spinner.Stop() + /* Install all the parts of the component */ + if hasImages { + pushImagesToRegistry(tempPath, component.Images, addShasumToImgs) + } - for index, file := range component.Files { - spinner.Updatef("Loading %s", file.Target) - sourceFile := componentPath.files + "/" + strconv.Itoa(index) + if hasRepos { + pushReposToRepository(componentPath.repos, component.Repos) + } - // If a shasum is specified check it again on deployment as well - if file.Shasum != "" { - spinner.Updatef("Validating SHASUM for %s", file.Target) - utils.ValidateSha256Sum(file.Shasum, sourceFile) - } + if hasDataInjections { + waitGroup := sync.WaitGroup{} + defer waitGroup.Wait() + performDataInjections(&waitGroup, componentPath, component.DataInjections) + } - // Replace temp target directories - file.Target = strings.Replace(file.Target, "###ZARF_TEMP###", tempPath.base, 1) + if hasCharts || hasManifests { + installedCharts = installChartAndManifests(componentPath, component) + } - // Copy the file to the destination - spinner.Updatef("Saving %s", file.Target) - err := copy.Copy(sourceFile, file.Target) - if err != nil { - spinner.Fatalf(err, "Unable to copy the contents of %s", file.Target) - } + // Run the 'after' scripts after all other attributes of the component has been deployed + runComponentScripts(component.Scripts.After, component.Scripts) - // Loop over all symlinks and create them - for _, link := range file.Symlinks { - spinner.Updatef("Adding symlink %s->%s", link, file.Target) - // Try to remove the filepath if it exists - _ = os.RemoveAll(link) - // Make sure the parent directory exists - _ = utils.CreateFilePath(link) - // Create the symlink - err := os.Symlink(file.Target, link) - if err != nil { - spinner.Fatalf(err, "Unable to create the symbolic link %s -> %s", link, file.Target) - } - } + return installedCharts +} - // Cleanup now to reduce disk pressure - _ = os.RemoveAll(sourceFile) - } - spinner.Success() +// Run scripts that a component has provided +func runComponentScripts(scripts []string, componentScript types.ZarfComponentScripts) { + for _, script := range scripts { + loopScriptUntilSuccess(script, componentScript) } +} - if isSeedRegistry { - preSeedRegistry(tempPath) - valueTemplate = template.Generate() +// Move files onto the host of the machine performing the deployment +func processComponentFiles(componentFiles []types.ZarfFile, sourceLocation, tempPathBase string) { + var spinner message.Spinner + if len(componentFiles) > 0 { + spinner = *message.NewProgressSpinner("Copying %d files", len(componentFiles)) + defer spinner.Stop() } - if !valueTemplate.Ready() && (hasImages || hasCharts || hasManifests || hasRepos) { - // If we are touching K8s, make sure we can talk to it once per deployment - spinner := message.NewProgressSpinner("Loading the Zarf State from the Kubernetes cluster") - defer spinner.Stop() + for index, file := range componentFiles { + spinner.Updatef("Loading %s", file.Target) + sourceFile := path.Join(sourceLocation, strconv.Itoa(index)) + + // If a shasum is specified check it again on deployment as well + if file.Shasum != "" { + spinner.Updatef("Validating SHASUM for %s", file.Target) + utils.ValidateSha256Sum(file.Shasum, sourceFile) + } + + // Replace temp target directories + file.Target = strings.Replace(file.Target, "###ZARF_TEMP###", tempPathBase, 1) - state, err := k8s.LoadZarfState() + // Copy the file to the destination + spinner.Updatef("Saving %s", file.Target) + err := copy.Copy(sourceFile, file.Target) if err != nil { - spinner.Fatalf(err, "Unable to load the Zarf State from the Kubernetes cluster") + spinner.Fatalf(err, "Unable to copy the contents of %s", file.Target) } - if state.Distro == "" { - // If no distro the zarf secret did not load properly - spinner.Fatalf(nil, "Unable to load the zarf/zarf-state secret, did you remember to run zarf init first?") + // Loop over all symlinks and create them + for _, link := range file.Symlinks { + spinner.Updatef("Adding symlink %s->%s", link, file.Target) + // Try to remove the filepath if it exists + _ = os.RemoveAll(link) + // Make sure the parent directory exists + _ = utils.CreateFilePath(link) + // Create the symlink + err := os.Symlink(file.Target, link) + if err != nil { + spinner.Fatalf(err, "Unable to create the symbolic link %s -> %s", link, file.Target) + } } - // Continue loading state data if it is valid - config.InitState(state) - valueTemplate = template.Generate() + // Cleanup now to reduce disk pressure + _ = os.RemoveAll(sourceFile) + } + spinner.Success() - if hasImages && state.Architecture != config.GetArch() { - // If the package has images but the architectures don't match warn the user to avoid ugly hidden errors with image push/pull - spinner.Fatalf(nil, "This package architecture is %s, but this cluster seems to be initialized with the %s architecture", - config.GetArch(), - state.Architecture) - } +} - spinner.Success() +// Fetch the current ZarfState from the k8s cluster and generate a valueTemplate from the state values +func getUpdatedValueTemplate(component types.ZarfComponent) template.Values { + // If we are touching K8s, make sure we can talk to it once per deployment + spinner := message.NewProgressSpinner("Loading the Zarf State from the Kubernetes cluster") + defer spinner.Stop() + + state, err := k8s.LoadZarfState() + if err != nil { + spinner.Fatalf(err, "Unable to load the Zarf State from the Kubernetes cluster") } - if hasImages { - // Try image push up to 3 times - for retry := 0; retry < 3; retry++ { - if err := images.PushToZarfRegistry(tempPath.images, component.Images); err != nil { - message.Errorf(err, "Unable to push images to the Zarf Registry, retrying in 5 seconds...") - time.Sleep(5 * time.Second) - continue - } else { - break - } - } + if state.Distro == "" { + // If no distro the zarf secret did not load properly + spinner.Fatalf(nil, "Unable to load the zarf/zarf-state secret, did you remember to run zarf init first?") + } + // Continue loading state data if it is valid + config.InitState(state) + valueTemplate := template.Generate() + if len(component.Images) > 0 && state.Architecture != config.GetArch() { + // If the package has images but the architectures don't match warn the user to avoid ugly hidden errors with image push/pull + spinner.Fatalf(nil, "This package architecture is %s, but this cluster seems to be initialized with the %s architecture", + config.GetArch(), + state.Architecture) } - if hasRepos { - // Try repo push up to 3 times - for retry := 0; retry < 3; retry++ { - // Push all the repos from the extracted archive - if err := git.PushAllDirectories(componentPath.repos); err != nil { - message.Errorf(err, "Unable to push repos to the Zarf Repository, retrying in 5 seconds...") - time.Sleep(5 * time.Second) - continue - } else { - break - } + spinner.Success() + + return valueTemplate +} + +// Push all of the components images to the configured container registry +func pushImagesToRegistry(tempPath tempPaths, componentImages []string, addShasumToImg bool) { + if len(componentImages) == 0 { + return + } + + // Try image push up to 3 times + for retry := 0; retry < 3; retry++ { + if err := images.PushToZarfRegistry(tempPath.images, componentImages, addShasumToImg); err != nil { + message.Errorf(err, "Unable to push images to the Registry, retrying in 5 seconds...") + time.Sleep(5 * time.Second) + continue + } else { + break } } +} - // Start any data injection async - if hasDataInjections { - var waitGroup sync.WaitGroup +// Push all of the components git repos to the configured git server +func pushReposToRepository(reposPath string, repos []string) { + if len(repos) == 0 { + return + } - message.Info("Loading data injections") - for _, data := range component.DataInjections { - waitGroup.Add(1) - go handleDataInjection(&waitGroup, data, componentPath) + // Try repo push up to 3 times + for retry := 0; retry < 3; retry++ { + // Push all the repos from the extracted archive + if err := git.PushAllDirectories(reposPath); err != nil { + message.Errorf(err, "Unable to push repos to the Git Server, retrying in 5 seconds...") + time.Sleep(5 * time.Second) + continue + } else { + break } - defer waitGroup.Wait() } +} - if len(component.Charts) > 0 || len(component.Manifests) > 0 { - for _, chart := range component.Charts { - // zarf magic for the value file - for idx := range chart.ValuesFiles { - chartValueName := helm.StandardName(componentPath.values, chart) + "-" + strconv.Itoa(idx) - valueTemplate.Apply(component, chartValueName) - } +// Async'ly move data into a container running in a pod on the k8s cluster +func performDataInjections(waitGroup *sync.WaitGroup, componentPath componentPaths, dataInjections []types.ZarfDataInjection) { + if len(dataInjections) > 0 { + message.Info("Loading data injections") + } - // Generate helm templates to pass to gitops engine - addedConnectStrings, installedChartName := helm.InstallOrUpgradeChart(helm.ChartOptions{ - BasePath: componentPath.base, - Chart: chart, - Component: component, - }) - installedCharts = append(installedCharts, types.InstalledCharts{Namespace: chart.Namespace, ChartName: installedChartName}) - - // Iterate over any connectStrings and add to the main map - for name, description := range addedConnectStrings { - connectStrings[name] = description - } + for _, data := range dataInjections { + waitGroup.Add(1) + go handleDataInjection(waitGroup, data, componentPath) + } +} + +// Install all Helm charts and raw k8s manifests into the k8s cluster +func installChartAndManifests(componentPath componentPaths, component types.ZarfComponent) []types.InstalledChart { + installedCharts := []types.InstalledChart{} + + for _, chart := range component.Charts { + // zarf magic for the value file + for idx := range chart.ValuesFiles { + chartValueName := helm.StandardName(componentPath.values, chart) + "-" + strconv.Itoa(idx) + valueTemplate.Apply(component, chartValueName) } - for _, manifest := range component.Manifests { - for idx := range manifest.Kustomizations { - // Move kustomizations to files now - destination := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, idx) - manifest.Files = append(manifest.Files, destination) - } + // Generate helm templates to pass to gitops engine + addedConnectStrings, installedChartName := helm.InstallOrUpgradeChart(helm.ChartOptions{ + BasePath: componentPath.base, + Chart: chart, + Component: component, + }) + installedCharts = append(installedCharts, types.InstalledChart{Namespace: chart.Namespace, ChartName: installedChartName}) + + // Iterate over any connectStrings and add to the main map + for name, description := range addedConnectStrings { + connectStrings[name] = description + } + } - if manifest.Namespace == "" { - // Helm gets sad when you don't provide a namespace even though we aren't using helm templating - manifest.Namespace = corev1.NamespaceDefault - } + for _, manifest := range component.Manifests { + for idx := range manifest.Kustomizations { + // Move kustomizations to files now + destination := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, idx) + manifest.Files = append(manifest.Files, destination) + } - // Iterate over any connectStrings and add to the main map - addedConnectStrings, installedChartName := helm.GenerateChart(componentPath.manifests, manifest, component) - installedCharts = append(installedCharts, types.InstalledCharts{Namespace: manifest.Namespace, ChartName: installedChartName}) - for name, description := range addedConnectStrings { - connectStrings[name] = description - } + if manifest.Namespace == "" { + // Helm gets sad when you don't provide a namespace even though we aren't using helm templating + manifest.Namespace = corev1.NamespaceDefault + } + + // Iterate over any connectStrings and add to the main map + addedConnectStrings, installedChartName := helm.GenerateChart(componentPath.manifests, manifest, component) + installedCharts = append(installedCharts, types.InstalledChart{Namespace: manifest.Namespace, ChartName: installedChartName}) + + // Iterate over any connectStrings and add to the main map + for name, description := range addedConnectStrings { + connectStrings[name] = description } } - for _, script := range component.Scripts.After { - loopScriptUntilSuccess(script, component.Scripts) + return installedCharts +} + +func writeSBOMFiles(sbomViewFiles []string) error { + // Check if we even have any SBOM files to process + if len(sbomViewFiles) == 0 { + return nil } - if isSeedRegistry { - postSeedRegistry(tempPath) + // Cleanup any failed prior removals + _ = os.RemoveAll(config.ZarfSBOMDir) + + // Create the directory again + err := utils.CreateDirectory(config.ZarfSBOMDir, 0755) + if err != nil { + return err } - return installedCharts + // Write each of the sbom files + for _, file := range sbomViewFiles { + // Our file copy lib explodes on these files for some reason... + data, err := ioutil.ReadFile(file) + if err != nil { + message.Fatalf(err, "Unable to read the sbom-viewer file %s", file) + } + dst := filepath.Join(config.ZarfSBOMDir, filepath.Base(file)) + err = ioutil.WriteFile(dst, data, 0644) + if err != nil { + message.Debugf("Unable to write the sbom-viewer file %s", dst) + return err + } + } + + return nil +} + +func printTablesForDeployment(componentsToDeploy []types.ZarfComponent) { + // If not init config, print the application connection table + if !config.IsZarfInitConfig() { + message.PrintConnectStringTable(connectStrings) + } else { + // otherwise, print the init config connection and passwords + loginTableHeader := pterm.TableData{ + {" Application", "Username", "Password", "Connect"}, + } + + loginTable := pterm.TableData{} + if config.GetContainerRegistryInfo().InternalRegistry { + loginTable = append(loginTable, pterm.TableData{{" Registry", config.GetContainerRegistryInfo().PushUsername, config.GetContainerRegistryInfo().PushPassword, "zarf connect registry"}}...) + } + + for _, component := range componentsToDeploy { + // Show message if including logging stack + if component.Name == "logging" { + loginTable = append(loginTable, pterm.TableData{{" Logging", "zarf-admin", config.GetState().LoggingSecret, "zarf connect logging"}}...) + } + // Show message if including git-server + if component.Name == "git-server" { + loginTable = append(loginTable, pterm.TableData{ + {" Git", config.GetGitServerInfo().PushUsername, config.GetState().GitServer.PushPassword, "zarf connect git"}, + {" Git (read-only)", config.GetGitServerInfo().PullUsername, config.GetState().GitServer.PullPassword, "zarf connect git"}, + }...) + } + } + + if len(loginTable) > 0 { + loginTable = append(loginTableHeader, loginTable...) + _ = pterm.DefaultTable.WithHasHeader().WithData(loginTable).Render() + } + } } func packageUsesK8s(zarfPackage types.ZarfPackage) bool { diff --git a/src/internal/packager/injector.go b/src/internal/packager/injector.go index 8f341a80dc..7f1817db93 100644 --- a/src/internal/packager/injector.go +++ b/src/internal/packager/injector.go @@ -84,16 +84,23 @@ func runInjectionMadness(tempPath tempPaths) { _ = k8s.DeletePod(k8s.ZarfNamespace, "injector") // Update the podspec image path and use the first node found - pod := buildInjectionPod(node[0], image, envVars, payloadConfigmaps, sha256sum) + pod, err := buildInjectionPod(node[0], image, envVars, payloadConfigmaps, sha256sum) + if err != nil { + // Just debug log the output because failures just result in trying the next image + message.Debug(err) + continue + } // Create the pod in the cluster pod, err = k8s.CreatePod(pod) - - // Just debug log the output because failures just result in trying the next image - message.Debug(pod, err) + if err != nil { + // Just debug log the output because failures just result in trying the next image + message.Debug(pod, err) + continue + } // if no error, try and wait for a seed image to be present, return if successful - if err == nil && hasSeedImages(spinner) { + if hasSeedImages(spinner) { return } @@ -290,7 +297,7 @@ func buildEnvVars(tempPath tempPaths) ([]corev1.EnvVar, error) { } // buildInjectionPod return a pod for injection with the appropriate containers to perform the injection -func buildInjectionPod(node, image string, envVars []corev1.EnvVar, payloadConfigmaps []string, payloadShasum string) *corev1.Pod { +func buildInjectionPod(node, image string, envVars []corev1.EnvVar, payloadConfigmaps []string, payloadShasum string) (*corev1.Pod, error) { pod := k8s.GeneratePod("injector", k8s.ZarfNamespace) executeMode := int32(0777) seedImage := config.GetSeedImage() @@ -345,7 +352,12 @@ func buildInjectionPod(node, image string, envVars []corev1.EnvVar, payloadConfi }, } - // Container definition for the injector pod + // Create container definition for the injector pod + newHost, err := utils.SwapHostWithoutSha(seedImage, "127.0.0.1:5001") + if err != nil { + message.Errorf(err, "Unable to swap the host of the seedImage for the injector pod: %#v", err) + return nil, err + } pod.Spec.Containers = []corev1.Container{ { Name: "injector", @@ -359,7 +371,7 @@ func buildInjectionPod(node, image string, envVars []corev1.EnvVar, payloadConfi "/zarf-stage2/zarf-registry", "/zarf-stage2/seed-image.tar", seedImage, - utils.SwapHost(seedImage, "127.0.0.1:5001"), + newHost, }, // Shared mount between the init and regular containers @@ -430,5 +442,5 @@ func buildInjectionPod(node, image string, envVars []corev1.EnvVar, payloadConfi }) } - return pod + return pod, nil } diff --git a/src/internal/packager/seed.go b/src/internal/packager/seed.go index b5da8f2449..e04c207d3a 100644 --- a/src/internal/packager/seed.go +++ b/src/internal/packager/seed.go @@ -1,6 +1,7 @@ package packager import ( + "fmt" "time" "github.com/defenseunicorns/zarf/src/config" @@ -9,9 +10,10 @@ import ( "github.com/defenseunicorns/zarf/src/internal/message" "github.com/defenseunicorns/zarf/src/internal/pki" "github.com/defenseunicorns/zarf/src/internal/utils" + "github.com/defenseunicorns/zarf/src/types" ) -func preSeedRegistry(tempPath tempPaths) { +func seedZarfState(tempPath tempPaths) { message.Debugf("package.preSeedRegistry(%#v)", tempPath) var ( @@ -33,18 +35,16 @@ func preSeedRegistry(tempPath tempPaths) { } // Attempt to load an existing state prior to init + // NOTE: We are ignoring the error here because we don't really expect a state to exist yet spinner.Updatef("Checking cluster for existing Zarf deployment") - state, err := k8s.LoadZarfState() - if err != nil { - spinner.Errorf(err, "Unable to load existing Zarf state") - } + state, _ := k8s.LoadZarfState() - // If the state is invalid, assume this is a new cluster - if state.Secret == "" { + // If the distro isn't populated in the state, assume this is a new cluster + if state.Distro == "" { spinner.Updatef("New cluster, no prior Zarf deployments found") // If the K3s component is being deployed, skip distro detection - if config.DeployOptions.ApplianceMode { + if config.InitOptions.ApplianceMode { distro = k8s.DistroIsK3s state.ZarfAppliance = true } else { @@ -61,10 +61,9 @@ func preSeedRegistry(tempPath tempPaths) { } // Defaults - state.NodePort = "31999" - state.Secret = utils.RandomString(120) state.Distro = distro state.Architecture = config.GetArch() + state.LoggingSecret = utils.RandomString(config.ZarfGeneratedPasswordLen) // Setup zarf agent PKI state.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) @@ -105,20 +104,14 @@ func preSeedRegistry(tempPath tempPaths) { state.StorageClass = "hostpath" } - // CLI provided overrides that haven't been processed already - if config.DeployOptions.NodePort != "" { - state.NodePort = config.DeployOptions.NodePort - } - if config.DeployOptions.Secret != "" { - state.Secret = config.DeployOptions.Secret - } - if config.DeployOptions.StorageClass != "" { - state.StorageClass = config.DeployOptions.StorageClass + if config.InitOptions.StorageClass != "" { + state.StorageClass = config.InitOptions.StorageClass } - spinner.Success() + state.GitServer = fillInEmptyGitServerValues(config.InitOptions.GitServer) + state.RegistryInfo = fillInEmptyContainerRegistryValues(config.InitOptions.RegistryInfo) - runInjectionMadness(tempPath) + spinner.Success() // Save the state back to K8s if err := k8s.SaveZarfState(state); err != nil { @@ -129,19 +122,97 @@ func preSeedRegistry(tempPath tempPaths) { config.InitState(state) } -func postSeedRegistry(tempPath tempPaths) { +func postSeedRegistry(tempPath tempPaths) error { message.Debug("packager.postSeedRegistry(%#v)", tempPath) // Try to kill the injector pod now - _ = k8s.DeletePod(k8s.ZarfNamespace, "injector") + if err := k8s.DeletePod(k8s.ZarfNamespace, "injector"); err != nil { + return err + } // Remove the configmaps labelMatch := map[string]string{"zarf-injector": "payload"} - _ = k8s.DeleteConfigMapsByLabel(k8s.ZarfNamespace, labelMatch) + if err := k8s.DeleteConfigMapsByLabel(k8s.ZarfNamespace, labelMatch); err != nil { + return err + } // Remove the injector service - _ = k8s.DeleteService(k8s.ZarfNamespace, "zarf-injector") + if err := k8s.DeleteService(k8s.ZarfNamespace, "zarf-injector"); err != nil { + return err + } // Push the seed images into to Zarf registry - images.PushToZarfRegistry(tempPath.seedImage, []string{config.GetSeedImage()}) + err := images.PushToZarfRegistry(tempPath.seedImage, []string{config.GetSeedImage()}, false) + + return err +} + +func fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) types.RegistryInfo { + // Set default url if an external registry was not provided + if containerRegistry.Address == "" { + containerRegistry.InternalRegistry = true + containerRegistry.NodePort = config.ZarfInClusterContainerRegistryNodePort + containerRegistry.Address = fmt.Sprintf("http://%s:%d", config.IPV4Localhost, containerRegistry.NodePort) + } + + // Generate a push-user password if not provided by init flag + if containerRegistry.PushPassword == "" { + containerRegistry.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + } + + // Set pull-username if not provided by init flag + if containerRegistry.PullUsername == "" { + if containerRegistry.InternalRegistry { + containerRegistry.PullUsername = config.ZarfRegistryPullUser + } else { + // If this is an external registry and a pull-user wasn't provided, use the same credentials as the push user + containerRegistry.PullUsername = containerRegistry.PushUsername + } + } + if containerRegistry.PullPassword == "" { + if containerRegistry.InternalRegistry { + containerRegistry.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + } else { + // If this is an external registry and a pull-user wasn't provided, use the same credentials as the push user + containerRegistry.PullPassword = containerRegistry.PushPassword + } + } + + if containerRegistry.Secret == "" { + containerRegistry.Secret = utils.RandomString(config.ZarfGeneratedSecretLen) + } + + return containerRegistry +} + +// Fill in empty GitServerInfo values with the defaults +func fillInEmptyGitServerValues(gitServer types.GitServerInfo) types.GitServerInfo { + // Set default svc url if an external repository was not provided + if gitServer.Address == "" { + gitServer.Address = config.ZarfInClusterGitServiceURL + gitServer.InternalServer = true + } + + // Generate a push-user password if not provided by init flag + if gitServer.PushPassword == "" { + gitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + } + + // Set read-user information if using an internal repository, otherwise copy from the push-user + if gitServer.PullUsername == "" { + if gitServer.InternalServer { + gitServer.PullUsername = config.ZarfGitReadUser + } else { + gitServer.PullUsername = gitServer.PushUsername + } + } + if gitServer.PullPassword == "" { + if gitServer.InternalServer { + gitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + } else { + gitServer.PullPassword = gitServer.PushPassword + } + } + + return gitServer } diff --git a/src/internal/template/template.go b/src/internal/template/template.go index 817d3ab33c..560fe0e4be 100644 --- a/src/internal/template/template.go +++ b/src/internal/template/template.go @@ -34,8 +34,8 @@ func Generate() Values { state := config.GetState() generated.state = state - pushUser, errPush := utils.GetHtpasswdString(config.ZarfRegistryPushUser, config.GetSecret(config.StateRegistryPush)) - pullUser, errPull := utils.GetHtpasswdString(config.ZarfRegistryPullUser, config.GetSecret(config.StateRegistryPull)) + pushUser, errPush := utils.GetHtpasswdString(config.GetContainerRegistryInfo().PushUsername, config.GetContainerRegistryInfo().PushPassword) + pullUser, errPull := utils.GetHtpasswdString(config.GetContainerRegistryInfo().PullUsername, config.GetContainerRegistryInfo().PullPassword) if errPush != nil || errPull != nil { message.Debug(errPush, errPull) message.Fatal(nil, "Unable to define `htpasswd` string for the Zarf user") @@ -45,14 +45,14 @@ func Generate() Values { generated.seedRegistry = config.GetSeedRegistry() generated.registry = config.GetRegistry() - generated.secret.registryPush = config.GetSecret(config.StateRegistryPush) - generated.secret.registryPull = config.GetSecret(config.StateRegistryPull) - generated.secret.registrySecret = config.GetSecret(config.StateRegistrySecret) + generated.secret.registryPush = config.GetContainerRegistryInfo().PushPassword + generated.secret.registryPull = config.GetContainerRegistryInfo().PullPassword + generated.secret.registrySecret = config.GetContainerRegistryInfo().Secret - generated.secret.gitPush = config.GetSecret(config.StateGitPush) - generated.secret.gitPull = config.GetSecret(config.StateGitPull) + generated.secret.gitPush = state.GitServer.PushPassword + generated.secret.gitPull = state.GitServer.PullPassword - generated.secret.logging = config.GetSecret(config.StateLogging) + generated.secret.logging = state.LoggingSecret generated.agentTLS = state.AgentTLS @@ -60,7 +60,7 @@ func Generate() Values { } func (values Values) Ready() bool { - return values.secret.htpasswd != "" + return values.state.Distro != "" } func (values Values) GetRegistry() string { @@ -79,9 +79,10 @@ func (values Values) Apply(component types.ZarfComponent, path string) { builtinMap := map[string]string{ "STORAGE_CLASS": values.state.StorageClass, "REGISTRY": values.registry, - "NODEPORT": values.state.NodePort, + "NODEPORT": fmt.Sprintf("%d", values.state.RegistryInfo.NodePort), "REGISTRY_AUTH_PUSH": values.secret.registryPush, "REGISTRY_AUTH_PULL": values.secret.registryPull, + "GIT_PUSH": values.state.GitServer.PushUsername, "GIT_AUTH_PUSH": values.secret.gitPush, "GIT_AUTH_PULL": values.secret.gitPull, } diff --git a/src/internal/utils/image.go b/src/internal/utils/image.go index 729fd41ffd..9e148562b7 100644 --- a/src/internal/utils/image.go +++ b/src/internal/utils/image.go @@ -1,12 +1,62 @@ package utils -import "regexp" +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "regexp" + "strings" +) -// For further explanation see https://regex101.com/library/PiL191 and https://regex101.com/r/PiL191/1 -var hostParser = regexp.MustCompile(`(?im)^([a-z0-9\-.]+\.[a-z0-9\-]+:?[0-9]*)?/?(.+)$`) +// For further explanation see https://regex101.com/library/4Rl8mW and https://regex101.com/r/4Rl8mW/1 +var hostParser = regexp.MustCompile(`(?im)([a-z0-9\-\_.]+)?(\/[a-z0-9\-.]+)?(:[\w\.\-\_]+)?$`) -// SwapHost Perform base url replacment without the docker libs -func SwapHost(src string, targetHost string) string { - var substitution = targetHost + "/$2" - return hostParser.ReplaceAllString(src, substitution) +// SwapHost Perform base url replacement and adds a sha1sum of the original url to the end of the src +func SwapHost(src string, targetHost string) (string, error) { + targetImage, err := getTargetImageFromURL(src) + return targetHost + "/" + targetImage, err +} + +func getTargetImageFromURL(src string) (string, error) { + submatches := hostParser.FindStringSubmatch(src) + if len(submatches) == 0 { + return "", fmt.Errorf("unable to get the targetImage from the provided source: %s", src) + } + + // Combine (most) of the matches we obtained + lastElementIndex := len(submatches) - 1 + targetImage := "" + for _, match := range submatches[1:lastElementIndex] { + targetImage += match + } + + // Get a sha1sum of the src without a potential image tag + tagMatcher := regexp.MustCompile(`(?im)(:[\w\.\-\_]+)?$`) + srcWithoutTag := tagMatcher.ReplaceAllString(src, "") + hasher := sha1.New() + _, err := io.WriteString(hasher, srcWithoutTag) + if err != nil { + return "", fmt.Errorf("unable to get targetImage from the provided source: %w", err) + } + sha1Hash := hex.EncodeToString(hasher.Sum(nil)) + + // Ensure we add the sha1sum before we apply an image tag + if strings.HasPrefix(submatches[lastElementIndex], ":") { + targetImage += "-" + sha1Hash + submatches[lastElementIndex] + } else { + targetImage += submatches[lastElementIndex] + "-" + sha1Hash + } + + return targetImage, nil +} + +// SwapHostWithoutSha Perform base url replacement but avoids adding a sha1sum of the original url. +func SwapHostWithoutSha(src string, targetHost string) (string, error) { + submatches := hostParser.FindStringSubmatch(src) + if len(submatches) == 0 { + return "", fmt.Errorf("unable to get the targetImage from the provided source: %s", src) + } + + return targetHost + "/" + submatches[0], nil } diff --git a/src/internal/utils/network.go b/src/internal/utils/network.go index 22c4351797..ba364d90b2 100644 --- a/src/internal/utils/network.go +++ b/src/internal/utils/network.go @@ -20,6 +20,24 @@ func IsUrl(source string) bool { return err == nil && parsedUrl.Scheme != "" && parsedUrl.Host != "" } +// DoesHostnamesMatch returns a boolean indicating if the hostname of two different URLs are the same. +func DoesHostnamesMatch(url1 string, url2 string) (bool, error) { + parsedURL1, err := url.Parse(url1) + if err != nil { + message.Debugf("unable to parse the url (%s)", url1) + + return false, err + } + parsedURL2, err := url.Parse(url2) + if err != nil { + message.Debugf("unable to parse the url (%s)", url2) + + return false, err + } + + return parsedURL1.Hostname() == parsedURL2.Hostname(), nil +} + func Fetch(url string) io.ReadCloser { // Get the data resp, err := http.Get(url) diff --git a/src/test/e2e/22_git_and_flux_test.go b/src/test/e2e/22_git_and_flux_test.go index fead25e9ce..6b784e29fe 100644 --- a/src/test/e2e/22_git_and_flux_test.go +++ b/src/test/e2e/22_git_and_flux_test.go @@ -56,9 +56,9 @@ func testGitServerReadOnly(t *testing.T, gitURL string) { config.InitState(state) // Get the repo as the readonly user - repoName := "mirror__repo1.dso.mil__platform-one__big-bang__apps__security-tools__twistlock" - getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitURL, config.ZarfGitPushUser, repoName), nil) - getRepoResponseBody, err := git.DoHttpThings(getRepoRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull)) + repoName := "twistlock-327c7a99d77a530fe94872911f0dabef839441bf" + getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitURL, config.GetGitServerInfo().PushUsername, repoName), nil) + getRepoResponseBody, err := git.DoHttpThings(getRepoRequest, config.ZarfGitReadUser, config.GetGitServerInfo().PullPassword) assert.NoError(t, err) // Make sure the only permissions are pull (read) @@ -74,15 +74,13 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Init the state variable state, err := k8s.LoadZarfState() require.NoError(t, err, "Failed to load Zarf state") - config.InitState(state) - - repoName := "mirror__github.com__defenseunicorns__zarf" + repoName := "zarf-bf89aea1b43dd0ea83360d4a219643a4bc8424c6" // Get the Zarf repo tag repoTag := "v0.15.0" getRepoTagsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/tags/%s", gitURL, config.ZarfGitPushUser, repoName, repoTag), nil) - getRepoTagsResponseBody, err := git.DoHttpThings(getRepoTagsRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull)) + getRepoTagsResponseBody, err := git.DoHttpThings(getRepoTagsRequest, config.ZarfGitReadUser, config.GetGitServerInfo().PullPassword) assert.NoError(t, err) // Make sure the pushed tag exists @@ -93,7 +91,7 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Get the Zarf repo commit repoHash := "c74e2e9626da0400e0a41e78319b3054c53a5d4e" getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/commits", gitURL, config.ZarfGitPushUser, repoName), nil) - getRepoCommitsResponseBody, err := git.DoHttpThings(getRepoCommitsRequest, config.ZarfGitReadUser, config.GetSecret(config.StateGitPull)) + getRepoCommitsResponseBody, err := git.DoHttpThings(getRepoCommitsRequest, config.ZarfGitReadUser, config.GetGitServerInfo().PullPassword) assert.NoError(t, err) // Make sure the pushed commit exists diff --git a/src/test/e2e/common.go b/src/test/e2e/common.go index 766524b05b..b9e59ecd0d 100644 --- a/src/test/e2e/common.go +++ b/src/test/e2e/common.go @@ -17,7 +17,7 @@ type ZarfE2ETest struct { } // getCLIName looks at the OS and CPU architecture to determine which Zarf binary needs to be run -func getCLIName() string { +func GetCLIName() string { var binaryName string if runtime.GOOS == "linux" { binaryName = "zarf" diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index a0d8e1ca36..95fb198731 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -44,7 +44,7 @@ func doAllTheThings(m *testing.M) (int, error) { // Set up constants in the global variable that all the tests are able to access e2e.arch = config.GetArch() - e2e.zarfBinPath = path.Join("build", getCLIName()) + e2e.zarfBinPath = path.Join("build", GetCLIName()) e2e.applianceMode = os.Getenv(applianceModeEnvVar) == "true" // Validate that the Zarf binary exists. If it doesn't that means the dev hasn't built it, usually by running diff --git a/src/test/external-test/README.md b/src/test/external-test/README.md new file mode 100644 index 0000000000..ca13349cc0 --- /dev/null +++ b/src/test/external-test/README.md @@ -0,0 +1,26 @@ +# Test Initializing Zarf w/ An External Git Repository and A External Container Registry +> Note: For this test case, we deploy an 'external' Git server and container registry as pods running within the k8s cluster. These are still considered 'external' servers since they already existed inside the k8s cluster before `zarf init` command is executed + +This directory holds the tests that verify Zarf can initialize a cluster to use an already existing Git server and container registry that is external to the resources Zarf manages. The tests in this directory are currently only run when manually executed. + + +## Running Tests Locally + +### Dependencies +Running the tests locally have the same prerequisites as running and building Zarf: +1. GoLang >= `1.19.x` +2. Make +3. Access to a cluster to test against + +### Actually Running The Test +Here are a few different ways to run the tests, based on your specific situation: + +```shell +# The default way, from the root directory of the repo. This will automatically build any Zarf related resources if they don't already exist (i.e. binary, init-package, example packages): +make test-external +``` + +```shell +# If you are in the root folder of the repository and already have everything built (i.e., the binary, the init-package and the flux-test example package): +go test ./src/test/external-git/... +``` diff --git a/src/test/external-test/docker-registry-values.yaml b/src/test/external-test/docker-registry-values.yaml new file mode 100644 index 0000000000..4edec32099 --- /dev/null +++ b/src/test/external-test/docker-registry-values.yaml @@ -0,0 +1,19 @@ +image: + repository: registry + tag: 2.7.1 + pullPolicy: IfNotPresent +imagePullSecrets: + - name: private-registry + +service: + name: registry + # type: ClusterIP + port: 5000 + annotations: {} + type: NodePort + nodePort: 31999 + +# Note: Super fake and not real htpasswd for testing purposes +secrets: + haSharedSecret: "" + htpasswd: "push-user:$2a$10$bnke4DeqY4qsAWIRsJCFluayx4v56mxhp/bfwubt.K83fudDzKAue" diff --git a/src/test/external-test/external_init_test.go b/src/test/external-test/external_init_test.go new file mode 100644 index 0000000000..ad8430947a --- /dev/null +++ b/src/test/external-test/external_init_test.go @@ -0,0 +1,92 @@ +package external_test + +import ( + "context" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/defenseunicorns/zarf/src/internal/utils" + test "github.com/defenseunicorns/zarf/src/test/e2e" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExternalDeploy(t *testing.T) { + zarfBinPath := path.Join("../../../build", test.GetCLIName()) + + // Install a gitea chart to the k8s cluster to act as the 'remote' git server + giteaChartURL := "https://dl.gitea.io/charts/gitea-5.0.8.tgz" + helmInstallArgs := []string{"install", "gitea", giteaChartURL, "-f", "gitea-values.yaml", "-n", "git-server", "--create-namespace"} + _, _, err := utils.ExecCommandWithContext(context.TODO(), true, "helm", helmInstallArgs...) + require.NoError(t, err, "unable to install gitea chart") + + // Install docker-registry chart to the k8s cluster to act as the 'remote' container registry + helmAddArgs := []string{"repo", "add", "twuni", "https://helm.twun.io"} + _, _, err = utils.ExecCommandWithContext(context.TODO(), true, "helm", helmAddArgs...) + require.NoError(t, err, "unable to add the docker-registry chart repo") + helmInstallArgs = []string{"install", "external-registry", "twuni/docker-registry", "-f=docker-registry-values.yaml", "-n=external-registry", "--create-namespace"} + _, _, err = utils.ExecCommandWithContext(context.TODO(), true, "helm", helmInstallArgs...) + require.NoError(t, err, "unable to install the docker-registry chart") + + // Verify the registry and gitea helm charts installed successfully + registryWaitCmd := []string{"wait", "deployment", "-n=external-registry", "external-registry-docker-registry", "--for", "condition=Available=True", "--timeout=5s"} + registryErrStr := "unable to verify the docker-registry chart installed successfully" + giteaWaitCmd := []string{"wait", "pod", "-n=git-server", "gitea-0", "--for", "condition=Ready=True", "--timeout=5s"} + giteaErrStr := "unable to verify the gitea chart installed successfully" + success := verifyKubectlWaitSuccess(t, 2, registryWaitCmd, registryErrStr) + require.True(t, success, registryErrStr) + success = verifyKubectlWaitSuccess(t, 2, giteaWaitCmd, giteaErrStr) + require.True(t, success, giteaErrStr) + + // Use Zarf to initialize the cluster + initArgs := []string{"init", + "--git-push-username=git-user", + "--git-push-password=superSecurePassword", + "--git-url=http://gitea-http.git-server.svc.cluster.local:3000", + "--registry-push-username=push-user", + "--registry-push-password=superSecurePassword", + "--registry-url=http://external-registry-docker-registry.external-registry.svc.cluster.local:5000", + "--nodeport=31999", + "--confirm"} + _, _, err = utils.ExecCommandWithContext(context.TODO(), true, zarfBinPath, initArgs...) + require.NoError(t, err, "unable to initialize the k8s server with zarf") + + // Deploy the flux example package + deployArgs := []string{"package", "deploy", "../../../build/zarf-package-flux-test-amd64.tar.zst", "--confirm", "-l=trace"} + _, _, err = utils.ExecCommandWithContext(context.TODO(), true, zarfBinPath, deployArgs...) + require.NoError(t, err, "unable to deploy flux example package") + + // Verify flux was able to pull from the 'external' repository + podinfoWaitCmd := []string{"wait", "deployment", "-n=podinfo", "podinfo", "--for", "condition=Available=True", "--timeout=3s"} + errorStr := "unable to verify flux deployed the podinfo example" + success = verifyKubectlWaitSuccess(t, 2, podinfoWaitCmd, errorStr) + assert.True(t, success, errorStr) +} + +func verifyKubectlWaitSuccess(t *testing.T, timeoutMinutes time.Duration, waitCmd []string, errorStr string) bool { + timeout := time.After(timeoutMinutes * time.Minute) + for { + // delay check 3 seconds + time.Sleep(3 * time.Second) + select { + // on timeout abort + case <-timeout: + t.Error(errorStr) + + // after delay, try running + default: + // Check that flux deployed the podinfo example + kubectlOut, err := exec.Command("kubectl", waitCmd...).Output() + // Log error + if err != nil { + t.Log(string(kubectlOut), err) + } + if strings.Contains(string(kubectlOut), "condition met") { + return true + } + } + } +} diff --git a/src/test/external-test/gitea-values.yaml b/src/test/external-test/gitea-values.yaml new file mode 100644 index 0000000000..08f0393688 --- /dev/null +++ b/src/test/external-test/gitea-values.yaml @@ -0,0 +1,34 @@ +persistence: + storageClass: "local-path" # 'local-path' is for k3d, 'standard' is for kind +gitea: + admin: + username: "git-user" + password: "superSecurePassword" # Note: Super fake and not real username/password for testing purposes + email: "zarf@localhost" + config: + APP_NAME: "Gitops Service" + server: + DISABLE_SSH: true + OFFLINE_MODE: true + database: + DB_TYPE: sqlite3 + security: + INSTALL_LOCK: true + service: + DISABLE_REGISTRATION: true + repository: + ENABLE_PUSH_CREATE_USER: true + FORCE_PRIVATE: true +resources: + requests: + cpu: "200m" + memory: "512Mi" + limits: + cpu: "1" + memory: "2Gi" + +memcached: + enabled: false + +postgresql: + enabled: false diff --git a/src/test/external-test/secret.yaml b/src/test/external-test/secret.yaml new file mode 100644 index 0000000000..e30a5fee35 --- /dev/null +++ b/src/test/external-test/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: private-git-server + namespace: git-server +type: Opaque +data: + # Note: Super fake and not real username/password for testing purposes + username: Z2l0LXVzZXI= + password: c3VwZXJTZWN1cmVQYXNzd29yZA== diff --git a/src/types/k8s.go b/src/types/k8s.go index 60ccb0cfb3..fe71c1d357 100644 --- a/src/types/k8s.go +++ b/src/types/k8s.go @@ -6,11 +6,15 @@ type ZarfState struct { Distro string `json:"distro" jsonschema:"description=K8s distribution of the cluster Zarf was deployed to"` Architecture string `json:"architecture" jsonschema:"description=Machine architecture of the k8s node(s)"` StorageClass string `json:"storageClass" jsonschema:"Default StorageClass value Zarf uses for variable templating"` - Secret string `json:"secret"` - NodePort string `json:"nodePort"` AgentTLS GeneratedPKI `json:"agentTLS" jsonschema:"PKI certificate information for the agent pods Zarf manages"` + + GitServer GitServerInfo `json:"gitServer" jsonschema:"description=Information about the repository Zarf is configured to use"` + RegistryInfo RegistryInfo `json:"registryInfo" jsonschema:"description=Information about the registry Zarf is configured to use"` + LoggingSecret string `json:"loggingSecret" jsonschema:"description=Secret value that the internal Grafana server was seeded with"` } +// DeployedPackage contains information about a Zarf Package that has been deployed to a cluster +// This object is saved as the data of a k8s secret within the 'zarf' namespace (not as part of the ZarfState secret). type DeployedPackage struct { Name string `json:"name"` Data ZarfPackage `json:"data"` @@ -19,16 +23,43 @@ type DeployedPackage struct { DeployedComponents []DeployedComponent `json:"deployedComponents"` } +// DeployedComponent contains information about a Zarf Package Component that has been deployed to a cluster. type DeployedComponent struct { - Name string `json:"name"` - InstalledCharts []InstalledCharts `json:"installedCharts"` + Name string `json:"name"` + InstalledCharts []InstalledChart `json:"installedCharts"` } -type InstalledCharts struct { +type InstalledChart struct { Namespace string `json:"namespace"` ChartName string `json:"chartName"` } +// GitServerInfo contains information Zarf uses to communicate with a git repository to push/pull repositories to. +type GitServerInfo struct { + PushUsername string `json:"pushUsername" jsonschema:"description=Username of a user with push access to the git repository"` + PushPassword string `json:"pushPassword" jsonschema:"description=Password of a user with push access to the git repository"` + PullUsername string `json:"pullUsername" jsonschema:"description=Username of a user with pull-only access to the git repository. If not provided for an external repository than the push-user is used"` + PullPassword string `json:"pullPassword" jsonschema:"description=Password of a user with pull-only access to the git repository. If not provided for an external repository than the push-user is used"` + + Address string `json:"address" jsonschema:"description=URL address of the git server"` + InternalServer bool `json:"internalServer" jsonschema:"description=Indicates if we are using a git server that Zarf is directly managing"` +} + +// RegistryInfo contains information Zarf uses to communicate with a container registry to push/pull images. +type RegistryInfo struct { + PushUsername string `json:"pushUsername" jsonschema:"description=Username of a user with push access to the registry"` + PushPassword string `json:"pushPassword" jsonschema:"description=Password of a user with push access to the registry"` + PullUsername string `json:"pullUsername" jsonschema:"description=Username of a user with pull-only access to the registry. If not provided for an external registry than the push-user is used"` + PullPassword string `json:"pullPassword" jsonschema:"description=Password of a user with pull-only access to the registry. If not provided for an external registry than the push-user is used"` + + Address string `json:"address" jsonschema:"description=URL address of the registry"` + NodePort int `json:"nodePort" jsonschema:"description=Nodeport of the registry. Only needed if the registry is running inside the kubernetes cluster"` + InternalRegistry bool `json:"internalRegistry" jsonschema:"description=Indicates if we are using a registry that Zarf is directly managing"` + + Secret string `json:"secret" jsonschema:"description=Secret value that the registry was seeded with"` +} + +// GeneratedPKI type GeneratedPKI struct { CA []byte `json:"ca"` Cert []byte `json:"cert"` diff --git a/src/types/runtime.go b/src/types/runtime.go index d2a5014ad4..b472f7874d 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -2,36 +2,43 @@ package types // ZarfCommonOptions tracks the user-defined preferences used across commands. type ZarfCommonOptions struct { - Confirm bool `json:"confirm"` - TempDirectory string `json:"tempDirectory"` - SetVariables map[string]string `json:"setVariables"` + Confirm bool `json:"confirm" jsonschema:"description=Verify that Zarf should perform an action"` + TempDirectory string `json:"tempDirectory" jsonschema:"description=Location Zarf should use as a staging ground when managing files and images for package creation and deployment"` + SetVariables map[string]string `json:"setVariables" jsonschema:"description=Key-Value map of variable names and their corresponding values that will be used to template against the Zarf package being used"` } // ZarfDeployOptions tracks the user-defined preferences during a package deployment type ZarfDeployOptions struct { - PackagePath string `json:"packagePath"` - Components string `json:"components"` - SGetKeyPath string `json:"sGetKeyPath"` + PackagePath string `json:"packagePath" jsonschema:"description=Location where a Zarf package to deploy can be found"` + Components string `json:"components" jsonschema:"description=Comma separated list of optional components to deploy"` + SGetKeyPath string `json:"sGetKeyPath" jsonschema:"description=Location where the public key component of a cosign key-pair can be found"` +} +// ZarfInitOptions tracks the user-defined options during cluster initialization. +type ZarfInitOptions struct { // Zarf init is installing the k3s component - ApplianceMode bool `json:"applianceMode"` + ApplianceMode bool `json:"applianceMode" jsonschema:"description=Indicates if Zarf was initialized while deploying its own k8s cluster"` // Zarf init override options - StorageClass string `json:"storageClass"` - Secret string `json:"secret"` - NodePort string `json:"nodePort"` + StorageClass string `json:"storageClass" jsonschema:"description=StorageClass of the k8s cluster Zarf is initializing"` + + // Using a remote git server + GitServer GitServerInfo `json:"gitServer" jsonschema:"description=Information about the repository Zarf is going to be using"` + + RegistryInfo RegistryInfo `json:"registryInfo" jsonschema:"description=Information about the registry Zarf is going to be using"` } // ZarfCreateOptions tracks the user-defined options used to create the package. type ZarfCreateOptions struct { - SkipSBOM bool `json:"skipSBOM"` - ImageCachePath string `json:"imageCachePath"` - Insecure bool `json:"insecure"` - OutputDirectory string `json:"outputDirectory"` + SkipSBOM bool `json:"skipSBOM" jsonschema:"description=Disable the generation of SBOM materials during package creation"` + ImageCachePath string `json:"imageCachePath" jsonschema:"description=Path to where a .cache directory of cached image that were pulled down to create packages"` + Insecure bool `json:"insecure" jsonschema:"description=Disable the need for shasum validations when pulling down files from the internet"` + OutputDirectory string `json:"outputDirectory" jsonschema:"description=Location where the finalized Zarf package will be placed"` } type ConnectString struct { - Description string `json:"description"` - Url string `json:"url"` + Description string `json:"description" jsonschema:"description=Descriptive text that explains what the resource you would be connecting to is used for"` + Url string `json:"url" jsonschema:"description=URL path that gets appended to the k8s port-forward result"` } + type ConnectStrings map[string]ConnectString diff --git a/src/ui/lib/api-types.ts b/src/ui/lib/api-types.ts index 469810faec..62d8d907ff 100644 --- a/src/ui/lib/api-types.ts +++ b/src/ui/lib/api-types.ts @@ -409,9 +409,19 @@ export interface ZarfState { /** * K8s distribution of the cluster Zarf was deployed to */ - distro: string; - nodePort: string; - secret: string; + distro: string; + /** + * Information about the repository Zarf is configured to use + */ + gitServer: GitServerInfo; + /** + * Secret value that the internal Grafana server was seeded with + */ + loggingSecret: string; + /** + * Information about the registry Zarf is configured to use + */ + registryInfo: RegistryInfo; storageClass: string; /** * Indicates if Zarf was initialized while deploying its own k8s cluster @@ -425,9 +435,88 @@ export interface GeneratedPKI { key: string; } +/** + * Information about the repository Zarf is configured to use + */ +export interface GitServerInfo { + /** + * URL address of the git server + */ + address: string; + /** + * Indicates if we are using a git server that Zarf is directly managing + */ + internalServer: boolean; + /** + * Password of a user with pull-only access to the git repository. If not provided for an + * external repository than the push-user is used + */ + pullPassword: string; + /** + * Username of a user with pull-only access to the git repository. If not provided for an + * external repository than the push-user is used + */ + pullUsername: string; + /** + * Password of a user with push access to the git repository + */ + pushPassword: string; + /** + * Username of a user with push access to the git repository + */ + pushUsername: string; +} + +/** + * Information about the registry Zarf is configured to use + */ +export interface RegistryInfo { + /** + * URL address of the registry + */ + address: string; + /** + * Indicates if we are using a registry that Zarf is directly managing + */ + internalRegistry: boolean; + /** + * Nodeport of the registry. Only needed if the registry is running inside the kubernetes + * cluster + */ + nodePort: number; + /** + * Password of a user with pull-only access to the registry. If not provided for an external + * registry than the push-user is used + */ + pullPassword: string; + /** + * Username of a user with pull-only access to the registry. If not provided for an external + * registry than the push-user is used + */ + pullUsername: string; + /** + * Password of a user with push access to the registry + */ + pushPassword: string; + /** + * Username of a user with push access to the registry + */ + pushUsername: string; + /** + * Secret value that the registry was seeded with + */ + secret: string; +} + export interface ConnectString { + /** + * Descriptive text that explains what the resource you would be connecting to is used for + */ description: string; - url: string; + /** + * URL path that gets appended to the k8s port-forward result + */ + url: string; } export interface DeployedPackage { @@ -438,36 +527,64 @@ export interface DeployedPackage { } export interface DeployedComponent { - installedCharts: InstalledCharts[]; + installedCharts: InstalledChart[]; name: string; } -export interface InstalledCharts { +export interface InstalledChart { chartName: string; namespace: string; } export interface ZarfCommonOptions { - confirm: boolean; - setVariables: { [key: string]: string }; + /** + * Verify that Zarf should perform an action + */ + confirm: boolean; + /** + * Key-Value map of variable names and their corresponding values that will be used to + * template against the Zarf package being used + */ + setVariables: { [key: string]: string }; + /** + * Location Zarf should use as a staging ground when managing files and images for package + * creation and deployment + */ tempDirectory: string; } export interface ZarfCreateOptions { - imageCachePath: string; - insecure: boolean; + /** + * Path to where a .cache directory of cached image that were pulled down to create packages + */ + imageCachePath: string; + /** + * Disable the need for shasum validations when pulling down files from the internet + */ + insecure: boolean; + /** + * Location where the finalized Zarf package will be placed + */ outputDirectory: string; - skipSBOM: boolean; + /** + * Disable the generation of SBOM materials during package creation + */ + skipSBOM: boolean; } export interface ZarfDeployOptions { - applianceMode: boolean; - components: string; - nodePort: string; - packagePath: string; - secret: string; - sGetKeyPath: string; - storageClass: string; + /** + * Comma separated list of optional components to deploy + */ + components: string; + /** + * Location where a Zarf package to deploy can be found + */ + packagePath: string; + /** + * Location where the public key component of a cosign key-pair can be found + */ + sGetKeyPath: string; } // Converts JSON strings to/from your types @@ -745,8 +862,9 @@ const typeMap: any = { { json: "agentTLS", js: "agentTLS", typ: r("GeneratedPKI") }, { json: "architecture", js: "architecture", typ: "" }, { json: "distro", js: "distro", typ: "" }, - { json: "nodePort", js: "nodePort", typ: "" }, - { json: "secret", js: "secret", typ: "" }, + { json: "gitServer", js: "gitServer", typ: r("GitServerInfo") }, + { json: "loggingSecret", js: "loggingSecret", typ: "" }, + { json: "registryInfo", js: "registryInfo", typ: r("RegistryInfo") }, { json: "storageClass", js: "storageClass", typ: "" }, { json: "zarfAppliance", js: "zarfAppliance", typ: true }, ], false), @@ -755,6 +873,24 @@ const typeMap: any = { { json: "cert", js: "cert", typ: "" }, { json: "key", js: "key", typ: "" }, ], false), + "GitServerInfo": o([ + { json: "address", js: "address", typ: "" }, + { json: "internalServer", js: "internalServer", typ: true }, + { json: "pullPassword", js: "pullPassword", typ: "" }, + { json: "pullUsername", js: "pullUsername", typ: "" }, + { json: "pushPassword", js: "pushPassword", typ: "" }, + { json: "pushUsername", js: "pushUsername", typ: "" }, + ], false), + "RegistryInfo": o([ + { json: "address", js: "address", typ: "" }, + { json: "internalRegistry", js: "internalRegistry", typ: true }, + { json: "nodePort", js: "nodePort", typ: 0 }, + { json: "pullPassword", js: "pullPassword", typ: "" }, + { json: "pullUsername", js: "pullUsername", typ: "" }, + { json: "pushPassword", js: "pushPassword", typ: "" }, + { json: "pushUsername", js: "pushUsername", typ: "" }, + { json: "secret", js: "secret", typ: "" }, + ], false), "ConnectString": o([ { json: "description", js: "description", typ: "" }, { json: "url", js: "url", typ: "" }, @@ -766,10 +902,10 @@ const typeMap: any = { { json: "name", js: "name", typ: "" }, ], false), "DeployedComponent": o([ - { json: "installedCharts", js: "installedCharts", typ: a(r("InstalledCharts")) }, + { json: "installedCharts", js: "installedCharts", typ: a(r("InstalledChart")) }, { json: "name", js: "name", typ: "" }, ], false), - "InstalledCharts": o([ + "InstalledChart": o([ { json: "chartName", js: "chartName", typ: "" }, { json: "namespace", js: "namespace", typ: "" }, ], false), @@ -785,13 +921,9 @@ const typeMap: any = { { json: "skipSBOM", js: "skipSBOM", typ: true }, ], false), "ZarfDeployOptions": o([ - { json: "applianceMode", js: "applianceMode", typ: true }, { json: "components", js: "components", typ: "" }, - { json: "nodePort", js: "nodePort", typ: "" }, { json: "packagePath", js: "packagePath", typ: "" }, - { json: "secret", js: "secret", typ: "" }, { json: "sGetKeyPath", js: "sGetKeyPath", typ: "" }, - { json: "storageClass", js: "storageClass", typ: "" }, ], false), "Architecture": [ "amd64", diff --git a/zarf.yaml b/zarf.yaml index 83fdd918df..9330ad80db 100644 --- a/zarf.yaml +++ b/zarf.yaml @@ -10,16 +10,19 @@ components: import: path: packages/distros/k3s + # This package moves the injector & registries binaries - name: zarf-injector required: true import: path: packages/zarf-injector + # Creates the temporary seed-registry - name: zarf-seed-registry required: true import: path: packages/zarf-registry + # Creates the permanent registry - name: zarf-registry required: true import: