Feature: Support Using Zarf With an External Registry and Repository (#…

…754)

## Description
This PR introduces the ability to connect to an already existing (and
reachable) Container Registry and/or Git Repository during the `zarf
init` command.

Closes #570 (Support using an external git server)
Closes #560 (Support using an external registry)
This implementation will serve as a good midway point on having a fully
HA in-cluster registry #375.



## PR Feature List
- Added several flags to the `init` command to support using an external
git repository
- Added several flags to the `init` command to support using an external
container registry
- Update `zarf connect registry` to direct to `{HOST}/v2/_catalog` (this
was confusing some other people since it would originally seem like the
registry was returning an empty page)
- Add utility function to create a tunnel to a service URL
- Created slightly better regexp for replacing the host from a
`containerImage` url
- semi-refactored the `zarf package deploy` logic


## Breaking Changes List
- We are changing the structure of the names of repos & containers we
are pushing (we are simplifying the name and adding a sha1 hash of the
original name to the end of the name)

Co-authored-by: Wayne Starr <Racer159@users.noreply.github.com>
Co-authored-by: Megamind <882485+jeff-mccoy@users.noreply.github.com>

.github/workflows/test-external.yml

@@ -0,0 +1,73 @@
+name: test-external
+on:
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "**.jpg"
+      - "**.png"
+      - "**.gif"
+      - "**.svg"
+      - "adr/**"
+      - "docs/**"
+
+# Abort prior jobs in the same workflow / PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Dependency: Install Golang"
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+
+      - name: "Dependency: Install Docker Buildx"
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: "Dependency: Install K3d"
+        run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | sudo bash"
+
+      - name: "Dependency: K3d cluster init"
+        run: k3d cluster delete && k3d cluster create
+
+      - name: "Dependency: Install Helm"
+        run: "curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash"
+
+      - name: "Install Kubectl"
+        run: |
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          curl -LO "https://dl.k8s.io/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
+          chmod +x kubectl
+          mv ./kubectl /usr/local/bin/kubectl
+
+      - name: "Checkout Repo"
+        uses: actions/checkout@v3
+
+      - name: "Build CLI"
+        run: make build-cli-linux-amd ARCH=amd64
+
+      - name: "Zarf Agent: Login to Docker Hub"
+        uses: docker/login-action@v2
+        with:
+          username: zarfdev
+          password: ${{ secrets.ZARF_DEV_DOCKERHUB }}
+
+      - name: "Zarf Agent: Build and Publish the Image"
+        run: |
+          cp build/zarf build/zarf-linux-amd64
+          docker buildx build --push --platform linux/amd64 --tag zarfdev/agent:$GITHUB_SHA .
+
+      - name: "Make Init Package"
+        run: make init-package AGENT_IMAGE=zarfdev/agent:$GITHUB_SHA
+
+      - name: "Run Tests"
+        # NOTE: This test run will create its own K3d cluster. A single cluster will be used throughout the test run.
+        run: make test-external
+
+      - name: "Cleanup"
+        run: make destroy

Makefile

@@ -16,7 +16,7 @@ ifneq ($(UNAME_S),Linux)
 	endif
 endif
 
-AGENT_IMAGE ?= defenseunicorns/zarf-agent:v0.21.1
+AGENT_IMAGE ?= zarfdev/agent:a57bb136f21441c66630403412c6f03fc7f9cd49
 
 CLI_VERSION := $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion")
 BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)'
@@ -105,19 +105,20 @@ cve-report:
 	go run main.go tools sbom packages . -o json | grype -o template -t .hooks/grype.tmpl > build/zarf-known-cves.csv
 
 # Inject and deploy a new dev version of zarf agent for testing (should have an existing zarf agent deployemt)
-# @todo: find a clean way to support Kind or k3d: k3d image import $(tag)
+# @todo: find a clean way to dynamically support Kind or k3d:
+#        when using kind: kind load docker-image $(tag)
+#        when using k3d: k3d image import $(tag)
 dev-agent-image:
 	$(eval tag := defenseunicorns/dev-zarf-agent:$(shell date +%s))
 	$(eval arch := $(shell uname -m))
 	CGO_ENABLED=0 GOOS=linux go build -o build/zarf-linux-$(arch) main.go
 	DOCKER_BUILDKIT=1 docker build --tag $(tag) --build-arg TARGETARCH=$(arch) . && \
-	kind load docker-image zarf-agent:$(tag) && \
+	k3d image import $(tag) && \
 	kubectl -n zarf set image deployment/agent-hook server=$(tag)
 
 init-package: ## Create the zarf init package, macos "brew install coreutils" first
 	@test -s $(ZARF_BIN) || $(MAKE) build-cli
-
-	@test -s ./build/zarf-init-$(ARCH).tar.zst || $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm .
+	$(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm .
 
 ci-release: init-package ## Create the init package
 
@@ -140,10 +141,18 @@ build-examples:
 
 	@test -s ./build/zarf-package-compose-example-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/composable-packages -o build -a $(ARCH) --confirm
 
-	@test -s ./build/zarf-package-flux-test-${ARCH}.tar.zst || $(ZARF_BIN) package create examples/flux-test -o build -a $(ARCH) --confirm
+	@test -s ./build/zarf-package-flux-test-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/flux-test -o build -a $(ARCH) --confirm
 
 ## Run e2e tests. Will automatically build any required dependencies that aren't present.
 ## Requires an existing cluster for the env var APPLIANCE_MODE=true
 .PHONY: test-e2e
-test-e2e: init-package build-examples
+test-e2e: build-examples
+	@test -s ./build/zarf-init-$(ARCH).tar.zst || $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE=$(AGENT_IMAGE) --confirm .
+	@test -s ./build/zarf-init-$(ARCH).tar.zst || $(MAKE) init-package
 	cd src/test/e2e && go test -failfast -v -timeout 30m
+
+test-external:
+	@test -s $(ZARF_BIN) || $(MAKE) build-cli
+	@test -s ./build/zarf-init-$(ARCH).tar.zst || $(MAKE) init-package
+	@test -s ./build/zarf-package-flux-test-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/flux-test -o build -a $(ARCH) --confirm
+	cd src/test/external-test && go test -failfast -v -timeout 30m

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_init.md

@@ -10,20 +10,52 @@ If you do not have a k8s cluster already configured, this command will give you
 This command looks for a zarf-init package in the local directory that the command was executed from. If no package is found in the local directory and the Zarf CLI exists somewhere outside of the current directory, Zarf will failover and attempt to find a zarf-init package in the directory that the Zarf binary is located in.
 
 
+
+Example Usage:
+# Initializing without any optional components:
+zarf init
+
+# Initializing w/ Zarfs internal git server:
+zarf init --components=git-server
+
+# Initializing w/ Zarfs internal git server and PLG stack:
+zarf init --components=git-server,logging
+
+# Initializing w/ an internal registry but with a different nodeport:
+zarf init --nodeport=30333
+
+# Initializing w/ an external registry:
+zarf init --registry-push-password={PASSWORD} --registry-push-username={USERNAME} --registry-url={URL}
+
+# Initializing w/ an external git server:
+zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-url={URL}
+
+
+
 ```
 zarf init [flags]
 ```
 
 ### Options
 
 ```
-      --components string      Comma-separated list of components to install.
-      --confirm                Confirm the install without prompting
-  -h, --help                   help for init
-      --nodeport string        Nodeport to access the Zarf container registry. Between [30000-32767]
-      --secret string          Root secret value that is used to 'seed' other secrets
-      --storage-class string   Describe the StorageClass to be used
-      --tmpdir string          Specify the temporary directory to use for intermediate files
+      --components string               Comma-separated list of components to install.
+      --confirm                         Confirm the install without prompting
+      --git-pull-password string        Password for the pull-only user to access the git server
+      --git-pull-username string        Username for pull-only access to the git server
+      --git-push-password string        Password for the push-user to access the git server
+      --git-push-username string        Username to access to the git server Zarf is configured to use. User must be able to create repositories via 'git push' (default "zarf-git-user")
+      --git-url string                  External git server url to use for this Zarf cluster
+  -h, --help                            help for init
+      --nodeport int                    Nodeport to access a registry internal to the k8s cluster. Between [30000-32767]
+      --registry-pull-password string   Password for the pull-only user to access the registry
+      --registry-pull-username string   Username for pull-only access to the registry
+      --registry-push-password string   Password for the push-user to connect to the registry
+      --registry-push-username string   Username to access to the registry Zarf is configured to use (default "zarf-push")
+      --registry-secret string          Registry secret value
+      --registry-url string             External registry url address to use for this Zarf cluster
+      --storage-class string            Describe the StorageClass to be used
+      --tmpdir string                   Specify the temporary directory to use for intermediate files
 ```
 
 ### Options inherited from parent commands

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md

@@ -10,7 +10,8 @@ zarf prepare patch-git [HOST] [FILE] [flags]
 ### Options
 
 ```
-  -h, --help   help for patch-git
+      --git-account string   User or organization name for the git account that the repos are created under. (default "zarf-git-user")
+  -h, --help                 help for patch-git
 ```
 
 ### Options inherited from parent commands

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools.md

@@ -20,7 +20,7 @@ Collection of additional tools to make airgap easier
 
 * [zarf](zarf.md)	 - DevSecOps Airgap Toolkit
 * [zarf tools archiver](zarf_tools_archiver.md)	 - Compress/Decompress tools for Zarf packages
-* [zarf tools get-admin-password](zarf_tools_get-admin-password.md)	 - Returns the Zarf admin password for gitea read from the zarf-state secret in the zarf namespace
+* [zarf tools get-git-password](zarf_tools_get-git-password.md)	 - Returns the push user's password for the Git server
 * [zarf tools monitor](zarf_tools_monitor.md)	 - Launch K9s tool for managing K8s clusters
 * [zarf tools registry](zarf_tools_registry.md)	 - Collection of registry commands provided by Crane
 * [zarf tools sbom](zarf_tools_sbom.md)	 - SBOM tools provided by Anchore Syft

docs/4-user-guide/1-the-zarf-cli/100-cli-commands/zarf_tools_get-git-password.md

@@ -0,0 +1,30 @@
+## zarf tools get-git-password
+
+Returns the push user's password for the Git server
+
+### Synopsis
+
+Reads the password for a user with push access to the configured Git server from the zarf-state secret in the zarf namespace
+
+```
+zarf tools get-git-password [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for get-git-password
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --architecture string   Architecture for OCI images
+  -l, --log-level string      Log level when running Zarf. Valid options are: warn, info, debug, trace
+      --no-progress           Disable fancy UI progress bars, spinners, logos, etc.
+```
+
+### SEE ALSO
+
+* [zarf tools](zarf_tools.md)	 - Collection of additional tools to make airgap easier
+

examples/flux-test/zarf.yaml

@@ -26,6 +26,6 @@ components:
           - podinfo-source.yaml
           - podinfo-kustomization.yaml
     repos:
-      - https://github.com/stefanprodan/podinfo
+      - https://github.com/stefanprodan/podinfo.git
     images:
       - ghcr.io/stefanprodan/podinfo:6.1.6

go.mod

@@ -372,11 +372,7 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.23.0 // indirect
-	golang.design/x/clipboard v0.6.2 // indirect
 	golang.org/x/exp v0.0.0-20220823124025-807a23277127 // indirect
-	golang.org/x/exp/shiny v0.0.0-20220921164117-439092de6870 // indirect
-	golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
-	golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
 	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect