From 931a4075dd83770001b806bac3dc8df517b99a3e Mon Sep 17 00:00:00 2001 From: Andrew Roth Date: Mon, 18 Oct 2021 15:36:59 -0700 Subject: [PATCH 1/5] wip --- .gitlab-ci.yml | 8 +- .pre-commit-config.yaml | 15 ++++ .vscode/launch.json | 2 +- CONTRIBUTING.md | 2 +- README.md | 10 +-- assets/manifests/common/traefik-tls.yaml | 2 +- assets/manifests/logging/pgl-stack.yaml | 2 +- assets/misc/k9s-theme.yaml | 2 +- assets/scripts/k3s-remove.sh | 2 +- assets/scripts/k3s.service | 2 +- cli/.gitignore | 2 +- cli/.gitlab-ci.yml | 1 - cli/cmd/logo.go | 90 +++++++++---------- cli/config/config.go | 2 +- e2e.sh | 10 +-- examples/.gitignore | 2 +- examples/appliance/manifests/podinfo.yaml | 1 - examples/game/README.md | 2 +- examples/game/image/index.html | 4 +- .../manifests/minio-operator.yaml | 1 - .../manifests/twistlock.yaml | 2 +- examples/single-big-bang-package/zarf.yaml | 2 +- examples/tiny-kafka/README.md | 2 +- .../tiny-kafka/manifests/kafka-topic.yaml | 2 +- examples/tiny-kafka/manifests/kafka.yaml | 2 +- examples/tiny-kafka/manifests/operator.yaml | 4 +- 26 files changed, 94 insertions(+), 82 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46a309c009..17d40ca5be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,10 +9,10 @@ stages: only: - master - merge_requests - - tags + - tags compile: - only: + only: - branches - merge_requests - tags @@ -42,7 +42,7 @@ package: - echo "PACKAGE_JOB_ID=$CI_JOB_ID" >> job.env artifacts: reports: - dotenv: job.env + dotenv: job.env paths: - build/ @@ -54,7 +54,7 @@ e2e: artifacts: true tags: - bigbang - - packages + - packages before_script: - yum install openssh-clients -y - eval $(ssh-agent -s) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..fe36a249f5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: detect-aws-credentials + args: + - "--allow-missing-credentials" + - id: detect-private-key + exclude: '^examples/big-bang/template/bigbang/values.yaml$' + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] diff --git a/.vscode/launch.json b/.vscode/launch.json index 1fcd1d8055..6908e48e57 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,4 +19,4 @@ }, ] -} \ No newline at end of file +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 771ccf0b02..09b8db823e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,4 +50,4 @@ We're still working on building out the test suite. If you want to help check ou - [#99](https://github.com/defenseunicorns/zarf/issues/99): Writing additional tests - [#100](https://github.com/defenseunicorns/zarf/issues/100): Each test should be runnable locally - [#101](https://github.com/defenseunicorns/zarf/issues/101): Run E2E tests on multiple distros -- [#102](https://github.com/defenseunicorns/zarf/issues/102): Make the E2E tests more efficient \ No newline at end of file +- [#102](https://github.com/defenseunicorns/zarf/issues/102): Make the E2E tests more efficient diff --git a/README.md b/README.md index 9a5c0c564c..3ba027f88a 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,12 @@ General usage steps below. For various ways to use Zarf, see [the examples fold [![asciicast](https://asciinema.org/a/427846.svg)](https://asciinema.org/a/427846) ### 2. Create the zarf cluster -- Move the `zarf`, `zarf-init.tar.zst` files to the system you will install the cluster to. +- Move the `zarf`, `zarf-init.tar.zst` files to the system you will install the cluster to. - Login or sudo/su to root. - Run `./zarf init` and follow the wizard. [![asciicast](https://asciinema.org/a/427721.svg)](https://asciinema.org/a/427721) -### 3. Add resources to the zarf cluster +### 3. Add resources to the zarf cluster - Following step 1b, make any necessary edits to the `zarf.yaml` file. - Then run `./zarf package create` to produce an `zarf-package-*.tar.zst` package. - Move the `zarf-package` into the same folder on the running zarf cluster as in step 2a. @@ -43,10 +43,10 @@ This tool utilizes software pulled from multiple sources and _some_ of them requ - [Iron Bank](https://registry1.dso.mil/) : Platform One's authorized, hardened, and approved container repository. ([product](https://p1.dso.mil/#/products/iron-bank/) | [pages](https://ironbank.dso.mil/) | [register](https://login.dso.mil/register)) #### Local Environment -- MacOS or Linux Operating System +- MacOS or Linux Operating System - [`make`](https://www.gnu.org/software/make/) : We use Makefiles for build automation - [`vagrant`](https://www.vagrantup.com/) : Easy creation and management of clean dev/test environments -- [`go`](https://golang.org/) : The programming language. Right now we are using v1.16.x +- [`go`](https://golang.org/) : The programming language. Right now we are using v1.16.x --- @@ -126,7 +126,7 @@ All OS options: - centos7 - centos8 - ubuntu -- debian +- debian - rocky In less than a minute, you'll have a kubernetes cluster running all the pre-requisites needed to host and deploy multiple other downstream clusters. diff --git a/assets/manifests/common/traefik-tls.yaml b/assets/manifests/common/traefik-tls.yaml index bc953156d9..513e40405a 100644 --- a/assets/manifests/common/traefik-tls.yaml +++ b/assets/manifests/common/traefik-tls.yaml @@ -16,4 +16,4 @@ spec: redirectRegex: regex: ^http://(.*) replacement: https://${1} - permanent: true \ No newline at end of file + permanent: true diff --git a/assets/manifests/logging/pgl-stack.yaml b/assets/manifests/logging/pgl-stack.yaml index 65d76b2781..82409d7ad3 100644 --- a/assets/manifests/logging/pgl-stack.yaml +++ b/assets/manifests/logging/pgl-stack.yaml @@ -63,6 +63,6 @@ spec: extraVolumeMounts: - name: journal mountPath: /var/log/journal - readOnly: true + readOnly: true image: pullPolicy: Never diff --git a/assets/misc/k9s-theme.yaml b/assets/misc/k9s-theme.yaml index 273034207b..c8fa1c1104 100644 --- a/assets/misc/k9s-theme.yaml +++ b/assets/misc/k9s-theme.yaml @@ -108,4 +108,4 @@ k9s: bgColor: *background indicator: fgColor: *foreground - bgColor: *purple \ No newline at end of file + bgColor: *purple diff --git a/assets/scripts/k3s-remove.sh b/assets/scripts/k3s-remove.sh index 305bb443b8..175d56f86b 100644 --- a/assets/scripts/k3s-remove.sh +++ b/assets/scripts/k3s-remove.sh @@ -96,4 +96,4 @@ rm -f /usr/local/bin/k9s rm -f /usr/local/bin/k3s-remove.sh rm -fr zarf-pki -echo -e '\033[0m' \ No newline at end of file +echo -e '\033[0m' diff --git a/assets/scripts/k3s.service b/assets/scripts/k3s.service index 5747ec042b..a27ba9da0f 100644 --- a/assets/scripts/k3s.service +++ b/assets/scripts/k3s.service @@ -24,4 +24,4 @@ RestartSec=5s ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s server --write-kubeconfig-mode=700 \ No newline at end of file +ExecStart=/usr/local/bin/k3s server --write-kubeconfig-mode=700 diff --git a/cli/.gitignore b/cli/.gitignore index fe21c0d854..3e8e1c62be 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -7,4 +7,4 @@ build .env .image-cache/ .DS_Store -bundle/ \ No newline at end of file +bundle/ diff --git a/cli/.gitlab-ci.yml b/cli/.gitlab-ci.yml index fd5932e40c..786b2794ac 100644 --- a/cli/.gitlab-ci.yml +++ b/cli/.gitlab-ci.yml @@ -37,4 +37,3 @@ release-from-tag: - echo "Creating a release for $CI_COMMIT_TAG" - release-cli create --name "Zarf CLI ${CI_COMMIT_TAG}" --tag-name "${CI_COMMIT_TAG}" --description "CLI for Zarf" --assets-link "{\"name\":\"zarf\",\"url\":\"https://repo1.dso.mil/platform-one/big-bang/apps/product-tools/zarf/cli/-/jobs/${JOB_ID}/artifacts/raw/zarf\"}" - \ No newline at end of file diff --git a/cli/cmd/logo.go b/cli/cmd/logo.go index 7610a177aa..94897db6a8 100644 --- a/cli/cmd/logo.go +++ b/cli/cmd/logo.go @@ -4,52 +4,52 @@ package cmd func getLogo() string { var logo = ` -         *,                                                                               -         *(((&&&&&/*.                                                                     -          *(((((%&&&&&&&*,                                                                -           *(((((((&&&&&&&&&*              ,,*****,.                      **%&&&&&((((((  -            *(((((((((&&&&&&&@*    **@@@@@@&&&&&&&&&&@@@@@**         */&&&&&&((((((((((   -              *((((((///(&&&&&&@@@@&&&&@@@@@@@@@&&&&&&&&&&&&&&@/* *%&&&&&&/////((((((*    -                *(((///////&&&&&&&&&&&&&@@@@@@@@@&&&&&&&&&&&&&&&&&(%&&&/**///////(/*      -       */&&&&&&&&&&&&&&&&*/***&%&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&/*******///*          +         *,                                                                               +         *(((&&&&&/*.                                                                     +          *(((((%&&&&&&&*,                                                                +           *(((((((&&&&&&&&&*              ,,*****,.                      **%&&&&&((((((  +            *(((((((((&&&&&&&@*    **@@@@@@&&&&&&&&&&@@@@@**         */&&&&&&((((((((((   +              *((((((///(&&&&&&@@@@&&&&@@@@@@@@@&&&&&&&&&&&&&&@/* *%&&&&&&/////((((((*    +                *(((///////&&&&&&&&&&&&&@@@@@@@@@&&&&&&&&&&&&&&&&&(%&&&/**///////(/*      +       */&&&&&&&&&&&&&&&&*/***&%&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&/*******///*             *&%&&&&&&&&&&&&&&&&&&&&&&&***&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&****/&&&&&&&&&&&&(/* - */((((((((((((((///////******%%&&&&&&&&//&@@*&&&&&&&&&&&&&&&&&&&#%&&&&*/####(/////((((/. -     */((((((((((///////******%%%%%%%%%(##@@@//%&%%%%%%%&%&%%&&/(@@(/&&(***///////(((*    -          ***(((((/////********%%%%%%%/&%***/((%%%%%%%%%%%%%%%%(#&@*%/%%***/////**        -            *&&%%%%%%//*******%%%%%%%%@@****%%/%%%%%%%%%%%%%%%%%%***@@%%**(%%%&&*         -          *&&%%%%%(////******/(%%%%%%%@@@**@@&%%%%%%%%%%%%%%%%%(@@*%@@%%*****////%&*      -        *&%%%%%#////////***/////%%%%%%*@@@@@/%%%%%%%%%%%%%%%%%%%%@@@@%%*****///////((*    -       *%%%%((((///////*    *////(%%%%%%##%%%%%%%%%%%(%%%%*%%%%%%%%%%%*                   -      *(((((((/***            */////#%%%%%%%%%%#%%%%%%%%%%%%%%%%%%%%#*                    -                   %%(           ,*///((%%%%%%%%(**/#%%%##**/%%%%%*                       -                 %%%&&&&           *///*/(((((########//######**                          -                 %&&&&&*          *#######(((((((//////((((*                              -                                  ###%##############(((#####*                             -                   %@&&          *&#(%######*#########(#####/                             -                   /&&* ..       ,&#(/%####(*#########/#####/             #%@%&&&         -             **         &&     ./%##((*&####/(#######(#####*(*            %&&&&&&         -           *@%%@*             *&#####((((####*(#####(*###(*(##*              ,  %@&       -          *@%%%%*            *%######((((*%####/*((*%####/*(###*  *                       -         *@%%%%%%*      *##* **#(###((((///#*#*(((((/#**#((*(##**#,*/##*,    %@&&         -         *@%%%*%%%*  ****,*##/*#*##(((((((/(((((((((/(((*(((((###########*,  #&&#         -         *@%%%*(%%%/*   **######(#((..((((((((((((((((((*  ,*(#####(((((*,                -         *@%%%#(*%%%%*   ,**/####(* */(((((((((((((((((*     ,**,                         -          *@%%%*(/(%%%%/*     ******(((((((((((*(((((*                                    -           *@%%%#(((*/%%%%%%##%%*((((((((((((**((((*                                      -            *@%%%%*(((((((((((((((((((((((*/%*((*.             (&&&(                      -             ,*%%%%%%*((((((((((((((((**%%%**,                (&                          -                *%%%%%%%%%(/*****(#%%%%%**                      &%                        -                   ,**%%%%%%%%%%%%%***                                                    -                                                                                          -             ,((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((,           -                         .....(((((((/////////////////((((((((.....                       -                                                                                          -            ///////////////      ///////      *****************  ***************,         -                    ////.       ///  ////     *///          ***  ****                     -                 ////,         ///    ////    *///////////////.  /////**/******           -              /////          //////////////   *///      *///     ///*                     -           ./////////////// ////         ///  *///        ////   ///*                     -                                                                                          + */((((((((((((((///////******%%&&&&&&&&//&@@*&&&&&&&&&&&&&&&&&&&#%&&&&*/####(/////((((/. +     */((((((((((///////******%%%%%%%%%(##@@@//%&%%%%%%%&%&%%&&/(@@(/&&(***///////(((*    +          ***(((((/////********%%%%%%%/&%***/((%%%%%%%%%%%%%%%%(#&@*%/%%***/////**        +            *&&%%%%%%//*******%%%%%%%%@@****%%/%%%%%%%%%%%%%%%%%%***@@%%**(%%%&&*         +          *&&%%%%%(////******/(%%%%%%%@@@**@@&%%%%%%%%%%%%%%%%%(@@*%@@%%*****////%&*      +        *&%%%%%#////////***/////%%%%%%*@@@@@/%%%%%%%%%%%%%%%%%%%%@@@@%%*****///////((*    +       *%%%%((((///////*    *////(%%%%%%##%%%%%%%%%%%(%%%%*%%%%%%%%%%%*                   +      *(((((((/***            */////#%%%%%%%%%%#%%%%%%%%%%%%%%%%%%%%#*                    +                   %%(           ,*///((%%%%%%%%(**/#%%%##**/%%%%%*                       +                 %%%&&&&           *///*/(((((########//######**                          +                 %&&&&&*          *#######(((((((//////((((*                              +                                  ###%##############(((#####*                             +                   %@&&          *&#(%######*#########(#####/                             +                   /&&* ..       ,&#(/%####(*#########/#####/             #%@%&&&         +             **         &&     ./%##((*&####/(#######(#####*(*            %&&&&&&         +           *@%%@*             *&#####((((####*(#####(*###(*(##*              ,  %@&       +          *@%%%%*            *%######((((*%####/*((*%####/*(###*  *                       +         *@%%%%%%*      *##* **#(###((((///#*#*(((((/#**#((*(##**#,*/##*,    %@&&         +         *@%%%*%%%*  ****,*##/*#*##(((((((/(((((((((/(((*(((((###########*,  #&&#         +         *@%%%*(%%%/*   **######(#((..((((((((((((((((((*  ,*(#####(((((*,                +         *@%%%#(*%%%%*   ,**/####(* */(((((((((((((((((*     ,**,                         +          *@%%%*(/(%%%%/*     ******(((((((((((*(((((*                                    +           *@%%%#(((*/%%%%%%##%%*((((((((((((**((((*                                      +            *@%%%%*(((((((((((((((((((((((*/%*((*.             (&&&(                      +             ,*%%%%%%*((((((((((((((((**%%%**,                (&                          +                *%%%%%%%%%(/*****(#%%%%%**                      &%                        +                   ,**%%%%%%%%%%%%%***                                                    +                                                                                          +             ,((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((,           +                         .....(((((((/////////////////((((((((.....                       +                                                                                          +            ///////////////      ///////      *****************  ***************,         +                    ////.       ///  ////     *///          ***  ****                     +                 ////,         ///    ////    *///////////////.  /////**/******           +              /////          //////////////   *///      *///     ///*                     +           ./////////////// ////         ///  *///        ////   ///*                     +                                                                                           ` return logo diff --git a/cli/config/config.go b/cli/config/config.go index 132a985fa4..8d0f30ac3b 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -91,4 +91,4 @@ func WriteConfig(path string) { if err != nil { logContext.Fatal("Unable to write the config file") } -} \ No newline at end of file +} diff --git a/e2e.sh b/e2e.sh index eff7d3e50a..b64d24eea0 100755 --- a/e2e.sh +++ b/e2e.sh @@ -43,7 +43,7 @@ beforeAll() { # Download the job artifacts _run "curl -fL https://repo1.dso.mil/platform-one/big-bang/apps/product-tools/zarf/-/jobs/${PACKAGE_JOB_ID}/artifacts/download -o artifact.zip && unzip -jo artifact.zip" - + # List the downloaded files _run "ls -lah" @@ -68,7 +68,7 @@ afterAll() { } loadZarfCA() { - # Get the ca file for curl to trust + # Get the ca file for curl to trust _run "sudo cat zarf-pki/zarf-ca.crt" > zarf-ca.crt } @@ -80,7 +80,7 @@ testPrepareCommands() { if [ $EXPECTED_SHASUM != $ZARF_SHASUM ]; then echo -e "${RED}zarf prepare sha256sum failed for local file${NOCOLOR}" exit 1 - fi + fi # Validate working SHASUM for remote asset EXPECTED_SHASUM="c3cdea0573ba5a058ec090b5d2683bf398e8b1614c37ec81136ed03b78167617" ZARF_SHASUM=$(_run "zarf prepare sha256sum https://zarf-public.s3-us-gov-west-1.amazonaws.com/pipelines/zarf-prepare-shasum-remote-test-file.txt") @@ -137,7 +137,7 @@ beforeAll testPrepareCommands -# Get the admin credentials +# Get the admin credentials ZARF_PWD=$(_run "sudo zarf tools get-admin-password") # Test that k9s is happy @@ -162,4 +162,4 @@ testDataInjection testGitBasedHelmChart # Perform final cleanup -afterAll \ No newline at end of file +afterAll diff --git a/examples/.gitignore b/examples/.gitignore index 1ca6d28d14..b7beff54c5 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1 +1 @@ -sync/ \ No newline at end of file +sync/ diff --git a/examples/appliance/manifests/podinfo.yaml b/examples/appliance/manifests/podinfo.yaml index 713332cd0e..3e9e70f724 100644 --- a/examples/appliance/manifests/podinfo.yaml +++ b/examples/appliance/manifests/podinfo.yaml @@ -24,4 +24,3 @@ spec: name: podinfo port: number: 9898 - diff --git a/examples/game/README.md b/examples/game/README.md index 0f611dd6fd..3c2e4a1b7f 100644 --- a/examples/game/README.md +++ b/examples/game/README.md @@ -1,6 +1,6 @@ ## Zarf Game Mode Example -This example demonstrates using Zarf to kill time (and evil). In this mode there is no gitops service and Zarf is simply a standard means of wrapping airgap concerns for K3s. Game mode is identical to [Appliance Mode](../appliance/README.md), but more fun. +This example demonstrates using Zarf to kill time (and evil). In this mode there is no gitops service and Zarf is simply a standard means of wrapping airgap concerns for K3s. Game mode is identical to [Appliance Mode](../appliance/README.md), but more fun. ### Steps to use: 1. Create a Zarf cluster as outlined in the main [README](../../README.md#2-create-the-zarf-cluster) diff --git a/examples/game/image/index.html b/examples/game/image/index.html index 5d0f4de51c..4040696d61 100644 --- a/examples/game/image/index.html +++ b/examples/game/image/index.html @@ -1,7 +1,7 @@ @@ -20,4 +20,4 @@ }); - \ No newline at end of file + diff --git a/examples/postgres-operator/manifests/minio-operator.yaml b/examples/postgres-operator/manifests/minio-operator.yaml index e8e39d4f77..2958091083 100644 --- a/examples/postgres-operator/manifests/minio-operator.yaml +++ b/examples/postgres-operator/manifests/minio-operator.yaml @@ -20,4 +20,3 @@ spec: limits: cpu: 200m memory: 256Mi - diff --git a/examples/single-big-bang-package/manifests/twistlock.yaml b/examples/single-big-bang-package/manifests/twistlock.yaml index 3a62a1695a..63c06ea78e 100644 --- a/examples/single-big-bang-package/manifests/twistlock.yaml +++ b/examples/single-big-bang-package/manifests/twistlock.yaml @@ -31,4 +31,4 @@ spec: service: name: twistlock-console port: - number: 8081 \ No newline at end of file + number: 8081 diff --git a/examples/single-big-bang-package/zarf.yaml b/examples/single-big-bang-package/zarf.yaml index e6f5a69912..172b416278 100644 --- a/examples/single-big-bang-package/zarf.yaml +++ b/examples/single-big-bang-package/zarf.yaml @@ -16,4 +16,4 @@ components: # https://umbrella-bigbang-releases.s3-us-gov-west-1.amazonaws.com/umbrella/1.14.0/images.txt images: - registry1.dso.mil/ironbank/twistlock/defender/defender:20.12.531 - - registry1.dso.mil/ironbank/twistlock/console/console:21.04.412 \ No newline at end of file + - registry1.dso.mil/ironbank/twistlock/console/console:21.04.412 diff --git a/examples/tiny-kafka/README.md b/examples/tiny-kafka/README.md index a9cb8362db..e112b14aab 100644 --- a/examples/tiny-kafka/README.md +++ b/examples/tiny-kafka/README.md @@ -13,4 +13,4 @@ Testing will require JDK and the kafka tools: `sudo apt install openjdk-14-jdk- 1. Install JDK and extract the Kafka tools from the package `/opt/kafka.tgz` 2. Get the Nodeport: `NODEPORT=$(kubectl get service demo-kafka-external-bootstrap -n kafka-demo -o=jsonpath='{.spec.ports[0].nodePort}{"\n"}')` 3. For pub: `./bin/kafka-console-producer.sh --broker-list localhost:$NODEPORT --topic cool-topic` -4. For sub: `./bin/kafka-console-consumer.sh --bootstrap-server localhost:$NODEPORT --topic cool-topic` \ No newline at end of file +4. For sub: `./bin/kafka-console-consumer.sh --bootstrap-server localhost:$NODEPORT --topic cool-topic` diff --git a/examples/tiny-kafka/manifests/kafka-topic.yaml b/examples/tiny-kafka/manifests/kafka-topic.yaml index 817bb09ece..bf33db8797 100644 --- a/examples/tiny-kafka/manifests/kafka-topic.yaml +++ b/examples/tiny-kafka/manifests/kafka-topic.yaml @@ -7,4 +7,4 @@ metadata: strimzi.io/cluster: "demo" spec: partitions: 3 - replicas: 1 \ No newline at end of file + replicas: 1 diff --git a/examples/tiny-kafka/manifests/kafka.yaml b/examples/tiny-kafka/manifests/kafka.yaml index 91c46d37ee..6f8d27a971 100644 --- a/examples/tiny-kafka/manifests/kafka.yaml +++ b/examples/tiny-kafka/manifests/kafka.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: name: kafka-demo ---- +--- apiVersion: kafka.strimzi.io/v1beta2 kind: Kafka metadata: diff --git a/examples/tiny-kafka/manifests/operator.yaml b/examples/tiny-kafka/manifests/operator.yaml index 8402c88ed7..304a331dcd 100644 --- a/examples/tiny-kafka/manifests/operator.yaml +++ b/examples/tiny-kafka/manifests/operator.yaml @@ -14,5 +14,5 @@ spec: valuesContent: |- imageRegistryOverride: registry1.dso.mil imageRepositoryOverride: ironbank/opensource/strimzi - watchNamespaces: - - kafka-demo \ No newline at end of file + watchNamespaces: + - kafka-demo From 8eb6ac36d29dad503912bd8a54a425d2618d8073 Mon Sep 17 00:00:00 2001 From: Andrew Roth Date: Mon, 18 Oct 2021 15:43:07 -0700 Subject: [PATCH 2/5] wip --- .editorconfig | 7 +- .pre-commit-config.yaml | 8 + test/e2e/terraform_ssh_e2e_test.go | 350 ++++++++++++++--------------- 3 files changed, 187 insertions(+), 178 deletions(-) diff --git a/.editorconfig b/.editorconfig index d6c8e7d3b9..316e7aee92 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,8 +10,9 @@ trim_trailing_whitespace = true max_line_length = 120 tab_width = 4 +[{Makefile,go.mod,go.sum,*.go,.gitmodules}] +indent_style = tab +indent_size = 4 + [*.md] trim_trailing_whitespace = false - -[Makefile] -indent_style = tab diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe36a249f5..1e4705fcd3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,3 +13,11 @@ repos: - id: fix-byte-order-marker - id: trailing-whitespace args: [--markdown-linebreak-ext=md] + - repo: https://github.com/sirosen/fix-smartquotes + rev: 0.2.0 + hooks: + - id: fix-smartquotes + - repo: https://github.com/dnephin/pre-commit-golang + rev: v0.4.0 + hooks: + - id: go-fmt diff --git a/test/e2e/terraform_ssh_e2e_test.go b/test/e2e/terraform_ssh_e2e_test.go index 40e7603b76..bfedba3ed2 100644 --- a/test/e2e/terraform_ssh_e2e_test.go +++ b/test/e2e/terraform_ssh_e2e_test.go @@ -1,196 +1,196 @@ package test import ( - "bufio" - "encoding/base64" - "fmt" - "github.com/gruntwork-io/terratest/modules/random" - "github.com/gruntwork-io/terratest/modules/retry" - "github.com/stretchr/testify/require" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/gruntwork-io/terratest/modules/aws" - "github.com/gruntwork-io/terratest/modules/ssh" - "github.com/gruntwork-io/terratest/modules/terraform" - teststructure "github.com/gruntwork-io/terratest/modules/test-structure" + "bufio" + "encoding/base64" + "fmt" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/retry" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/gruntwork-io/terratest/modules/aws" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + teststructure "github.com/gruntwork-io/terratest/modules/test-structure" ) func TestTerraformSshExample(t *testing.T) { - t.Parallel() - - // Copy the terraform folder to a temp directory so we can run multiple tests in parallel - tmpFolder := teststructure.CopyTerraformFolderToTemp(t, "..", "tf/public-ec2-instance") - - // At the end of the test, run `terraform destroy` to clean up any resources that were created - defer teststructure.RunTestStage(t, "TEARDOWN", func() { - keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) - aws.DeleteEC2KeyPair(t, keyPair) - - terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) - terraform.Destroy(t, terraformOptions) - }) - - // Deploy the terraform infra - teststructure.RunTestStage(t, "SETUP", func() { - terraformOptions, keyPair, err := configureTerraformOptions(t, tmpFolder) - require.NoError(t, err) - - // Save the options and key pair so later test stages can use them - teststructure.SaveTerraformOptions(t, tmpFolder, terraformOptions) - teststructure.SaveEc2KeyPair(t, tmpFolder, keyPair) - - // This will run `terraform init` and `terraform apply` and fail the test if there are any errors - terraform.InitAndApply(t, terraformOptions) - }) - - // Upload the Zarf artifacts - teststructure.RunTestStage(t, "UPLOAD", func() { - terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) - keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) - - // This will upload the Zarf binary, init package, and other necessary files to the server so we can use them for - // tests - syncFilesToRemoteServer(t, terraformOptions, keyPair) - }) - - // Make sure we can SSH to the public Instance directly from the public Internet - teststructure.RunTestStage(t, "TEST", func() { - terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) - keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) - - // Finally run the actual test - test(t, terraformOptions, keyPair) - }) + t.Parallel() + + // Copy the terraform folder to a temp directory so we can run multiple tests in parallel + tmpFolder := teststructure.CopyTerraformFolderToTemp(t, "..", "tf/public-ec2-instance") + + // At the end of the test, run `terraform destroy` to clean up any resources that were created + defer teststructure.RunTestStage(t, "TEARDOWN", func() { + keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) + aws.DeleteEC2KeyPair(t, keyPair) + + terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) + terraform.Destroy(t, terraformOptions) + }) + + // Deploy the terraform infra + teststructure.RunTestStage(t, "SETUP", func() { + terraformOptions, keyPair, err := configureTerraformOptions(t, tmpFolder) + require.NoError(t, err) + + // Save the options and key pair so later test stages can use them + teststructure.SaveTerraformOptions(t, tmpFolder, terraformOptions) + teststructure.SaveEc2KeyPair(t, tmpFolder, keyPair) + + // This will run `terraform init` and `terraform apply` and fail the test if there are any errors + terraform.InitAndApply(t, terraformOptions) + }) + + // Upload the Zarf artifacts + teststructure.RunTestStage(t, "UPLOAD", func() { + terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) + keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) + + // This will upload the Zarf binary, init package, and other necessary files to the server so we can use them for + // tests + syncFilesToRemoteServer(t, terraformOptions, keyPair) + }) + + // Make sure we can SSH to the public Instance directly from the public Internet + teststructure.RunTestStage(t, "TEST", func() { + terraformOptions := teststructure.LoadTerraformOptions(t, tmpFolder) + keyPair := teststructure.LoadEc2KeyPair(t, tmpFolder) + + // Finally run the actual test + test(t, terraformOptions, keyPair) + }) } func configureTerraformOptions(t *testing.T, tmpFolder string) (*terraform.Options, *aws.Ec2Keypair, error) { - // A unique ID we can use to namespace resources so we don't clash with anything already in the AWS account or - // tests running in parallel - uniqueID := random.UniqueId() - namespace := "zarf" - stage := "terratest" - name := fmt.Sprintf("e2e-%s", uniqueID) - - // Get the region to use from the system's environment - awsRegion, err := getAwsRegion() - if err != nil { - return nil, nil, err - } - - // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked - instanceType := aws.GetRecommendedInstanceType(t, awsRegion, []string{"t3a.large", "t3.large", "t2.large"}) - - // Create an EC2 KeyPair that we can use for SSH access - keyPairName := fmt.Sprintf("%s-%s-%s", namespace, stage, name) - keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, keyPairName) - - // Construct the terraform options with default retryable errors to handle the most common retryable errors in - // terraform testing. - terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ - // The path to where our Terraform code is located - TerraformDir: tmpFolder, - - // Variables to pass to our Terraform code using -var options - Vars: map[string]interface{}{ - "aws_region": awsRegion, - "namespace": namespace, - "stage": stage, - "name": name, - "instance_type": instanceType, - "key_pair_name": keyPairName, - }, - }) - - return terraformOptions, keyPair, nil + // A unique ID we can use to namespace resources so we don't clash with anything already in the AWS account or + // tests running in parallel + uniqueID := random.UniqueId() + namespace := "zarf" + stage := "terratest" + name := fmt.Sprintf("e2e-%s", uniqueID) + + // Get the region to use from the system's environment + awsRegion, err := getAwsRegion() + if err != nil { + return nil, nil, err + } + + // Some AWS regions are missing certain instance types, so pick an available type based on the region we picked + instanceType := aws.GetRecommendedInstanceType(t, awsRegion, []string{"t3a.large", "t3.large", "t2.large"}) + + // Create an EC2 KeyPair that we can use for SSH access + keyPairName := fmt.Sprintf("%s-%s-%s", namespace, stage, name) + keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, keyPairName) + + // Construct the terraform options with default retryable errors to handle the most common retryable errors in + // terraform testing. + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: tmpFolder, + + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "aws_region": awsRegion, + "namespace": namespace, + "stage": stage, + "name": name, + "instance_type": instanceType, + "key_pair_name": keyPairName, + }, + }) + + return terraformOptions, keyPair, nil } func syncFilesToRemoteServer(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair) { - // Run `terraform output` to get the value of an output variable - publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") - - // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", - // as we know the Instance is running an Ubuntu AMI that has such a user - publicHost := ssh.Host{ - Hostname: publicInstanceIP, - SshKeyPair: keyPair.KeyPair, - SshUserName: "ubuntu", - } - - // It can take a minute or so for the Instance to boot up, so retry a few times - maxRetries := 15 - timeBetweenRetries, err := time.ParseDuration("5s") - require.NoError(t, err) - - // Wait for the instance to be ready - _, err = retry.DoWithRetryE(t, "Wait for the instance to be ready", maxRetries, timeBetweenRetries, func() (string, error){ - _, err := ssh.CheckSshCommandE(t, publicHost, "whoami") - if err != nil { - return "", err - } - return "", nil - }) - require.NoError(t, err) - - // Upload the compiled Zarf binary to the server. The ssh lib only supports sending strings so we'll base64encode it - // first - f, err := os.Open("../../build/zarf") - require.NoError(t, err) - reader := bufio.NewReader(f) - content, err := ioutil.ReadAll(reader) - require.NoError(t, err) - encodedZarfBinary := base64.StdEncoding.EncodeToString(content) - err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf.b64", encodedZarfBinary) - require.NoError(t, err) - output, err := ssh.CheckSshCommandE(t, publicHost, "cd $HOME && sudo bash -c 'base64 -d zarf.b64 > /usr/local/bin/zarf && chmod 0777 /usr/local/bin/zarf'") - require.NoError(t, err, output) - - // Upload zarf-init.tar.zst - f, err = os.Open("../../build/zarf-init.tar.zst") - require.NoError(t, err) - reader = bufio.NewReader(f) - content, err = ioutil.ReadAll(reader) - require.NoError(t, err) - encodedZarfInit := base64.StdEncoding.EncodeToString(content) - err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf-init.tar.zst.b64", encodedZarfInit) - require.NoError(t, err) - output, err = ssh.CheckSshCommandE(t, publicHost, "cd $HOME && base64 -d zarf-init.tar.zst.b64 > zarf-init.tar.zst") - require.NoError(t, err, output) + // Run `terraform output` to get the value of an output variable + publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") + + // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", + // as we know the Instance is running an Ubuntu AMI that has such a user + publicHost := ssh.Host{ + Hostname: publicInstanceIP, + SshKeyPair: keyPair.KeyPair, + SshUserName: "ubuntu", + } + + // It can take a minute or so for the Instance to boot up, so retry a few times + maxRetries := 15 + timeBetweenRetries, err := time.ParseDuration("5s") + require.NoError(t, err) + + // Wait for the instance to be ready + _, err = retry.DoWithRetryE(t, "Wait for the instance to be ready", maxRetries, timeBetweenRetries, func() (string, error) { + _, err := ssh.CheckSshCommandE(t, publicHost, "whoami") + if err != nil { + return "", err + } + return "", nil + }) + require.NoError(t, err) + + // Upload the compiled Zarf binary to the server. The ssh lib only supports sending strings so we'll base64encode it + // first + f, err := os.Open("../../build/zarf") + require.NoError(t, err) + reader := bufio.NewReader(f) + content, err := ioutil.ReadAll(reader) + require.NoError(t, err) + encodedZarfBinary := base64.StdEncoding.EncodeToString(content) + err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf.b64", encodedZarfBinary) + require.NoError(t, err) + output, err := ssh.CheckSshCommandE(t, publicHost, "cd $HOME && sudo bash -c 'base64 -d zarf.b64 > /usr/local/bin/zarf && chmod 0777 /usr/local/bin/zarf'") + require.NoError(t, err, output) + + // Upload zarf-init.tar.zst + f, err = os.Open("../../build/zarf-init.tar.zst") + require.NoError(t, err) + reader = bufio.NewReader(f) + content, err = ioutil.ReadAll(reader) + require.NoError(t, err) + encodedZarfInit := base64.StdEncoding.EncodeToString(content) + err = ssh.ScpFileToE(t, publicHost, 0644, "$HOME/zarf-init.tar.zst.b64", encodedZarfInit) + require.NoError(t, err) + output, err = ssh.CheckSshCommandE(t, publicHost, "cd $HOME && base64 -d zarf-init.tar.zst.b64 > zarf-init.tar.zst") + require.NoError(t, err, output) } func test(t *testing.T, terraformOptions *terraform.Options, keyPair *aws.Ec2Keypair) { - // Run `terraform output` to get the value of an output variable - publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") - - // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", - // as we know the Instance is running an Ubuntu AMI that has such a user - publicHost := ssh.Host{ - Hostname: publicInstanceIP, - SshKeyPair: keyPair.KeyPair, - SshUserName: "ubuntu", - } - - // Make sure `zarf --help` doesn't error - output, err := ssh.CheckSshCommandE(t, publicHost, "zarf --help") - require.NoError(t, err, output) - - // Test `zarf init just to make sure it returns a zero exit code.` - output, err = ssh.CheckSshCommandE(t, publicHost, "sudo bash -c 'zarf init --confirm --components management,logging,gitops-service --host localhost'") - require.NoError(t, err, output) + // Run `terraform output` to get the value of an output variable + publicInstanceIP := terraform.Output(t, terraformOptions, "public_instance_ip") + + // We're going to try to SSH to the instance IP, using the Key Pair we created earlier, and the user "ubuntu", + // as we know the Instance is running an Ubuntu AMI that has such a user + publicHost := ssh.Host{ + Hostname: publicInstanceIP, + SshKeyPair: keyPair.KeyPair, + SshUserName: "ubuntu", + } + + // Make sure `zarf --help` doesn't error + output, err := ssh.CheckSshCommandE(t, publicHost, "zarf --help") + require.NoError(t, err, output) + + // Test `zarf init just to make sure it returns a zero exit code.` + output, err = ssh.CheckSshCommandE(t, publicHost, "sudo bash -c 'zarf init --confirm --components management,logging,gitops-service --host localhost'") + require.NoError(t, err, output) } // getAwsRegion returns the desired AWS region to use by first checking the env var AWS_REGION, then checking //AWS_DEFAULT_REGION if AWS_REGION isn't set. If neither is set it returns an error func getAwsRegion() (string, error) { - val, present := os.LookupEnv("AWS_REGION") - if !present { - val, present = os.LookupEnv("AWS_DEFAULT_REGION") - } - if !present { - return "", fmt.Errorf("expected either AWS_REGION or AWS_DEFAULT_REGION env var to be set, but they were not") - } else { - return val, nil - } + val, present := os.LookupEnv("AWS_REGION") + if !present { + val, present = os.LookupEnv("AWS_DEFAULT_REGION") + } + if !present { + return "", fmt.Errorf("expected either AWS_REGION or AWS_DEFAULT_REGION env var to be set, but they were not") + } else { + return val, nil + } } From ee29e9f11fe842ed32e03c0d46ed7bcc3720cc45 Mon Sep 17 00:00:00 2001 From: Andrew Roth Date: Mon, 18 Oct 2021 16:32:33 -0700 Subject: [PATCH 3/5] wip --- .pre-commit-config.yaml | 10 ++++++++++ hooks/run-golangci-lint.sh | 2 ++ 2 files changed, 12 insertions(+) create mode 100755 hooks/run-golangci-lint.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e4705fcd3..670d45e527 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,3 +21,13 @@ repos: rev: v0.4.0 hooks: - id: go-fmt +# Commenting this one out for now since it fails +# - repo: local +# hooks: +# - id: golangci-lint +# name: golangci-lint +# entry: hooks/run-golangci-lint.sh +# types: [ go ] +# language: script +# pass_filenames: false +# description: "Runs `golangci-lint`, requires https://github.com/golangci/golangci-lint" diff --git a/hooks/run-golangci-lint.sh b/hooks/run-golangci-lint.sh new file mode 100755 index 0000000000..b7b9e8361c --- /dev/null +++ b/hooks/run-golangci-lint.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +cd cli && golangci-lint run "$@" From 26bb0680ba96375794dcfae9cbf7a0502ce69dc3 Mon Sep 17 00:00:00 2001 From: Andrew Roth Date: Mon, 18 Oct 2021 17:46:53 -0700 Subject: [PATCH 4/5] wip --- .pre-commit-config.yaml | 3 ++- CONTRIBUTING.md | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 670d45e527..77cb989ccc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: args: - "--allow-missing-credentials" - id: detect-private-key - exclude: '^examples/big-bang/template/bigbang/values.yaml$' + exclude: "^examples/big-bang/template/bigbang/values.yaml$" - id: end-of-file-fixer - id: fix-byte-order-marker - id: trailing-whitespace @@ -22,6 +22,7 @@ repos: hooks: - id: go-fmt # Commenting this one out for now since it fails +# Normally we wouldn't need to do a local hook but we need to modify the shell script that gets run to first change directories into the `cli` folder # - repo: local # hooks: # - id: golangci-lint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09b8db823e..ef2993feee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,19 @@ Here's what a typical "day in the life" of a Zarf developer might look like. Kee This section dives deeper into how we test Zarf +### Pre-Commit Hooks and Linting + +In this repo we use [pre-commit](https://pre-commit.com/) hooks for automated validation and linting. The CI pipeline will validate that all of the hooks pass so we strongly recommend that you install the hooks locally or you'll be spending a lot of time manually fixing issues that could be fixed automatically very quickly. + +#### Pre-Commit Prerequisites + +1. Install [pre-commit](https://pre-commit.com/) +1. Install [go](https://golang.org/) +1. Install [golangci-lint](https://github.com/golangci/golangci-lint) +1. Run `pre-commit install` in the repo to install the pre-commit hooks. This will make the hooks run automatically each time you `git commit`. If you want to skip the hooks for any reason you can run `git commit --no-verify` to skip them. + +> **HINT:** *Consider [automatically enabling the hooks in every Git repository](https://pre-commit.com/#automatically-enabling-pre-commit-on-repositories)* + ### End2End Testing Our E2E tests utilize [Terratest](https://terratest.gruntwork.io/). They create real infrastructure in AWS that the tests get run on. By doing this we are able to make the test environments ephemeral and allow them to be run in parallel so that we can do more testing more quickly. From b634a42f31b7b6f1594de2466eb38cfde9fa97a1 Mon Sep 17 00:00:00 2001 From: Andrew Roth Date: Mon, 18 Oct 2021 18:04:03 -0700 Subject: [PATCH 5/5] wip --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77cb989ccc..deb3b60e3d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: hooks: - id: go-fmt # Commenting this one out for now since it fails -# Normally we wouldn't need to do a local hook but we need to modify the shell script that gets run to first change directories into the `cli` folder +## Normally we wouldn't need to do a local hook but we need to modify the shell script that gets run to first change directories into the `cli` folder # - repo: local # hooks: # - id: golangci-lint