From c6d2ee2887c3ac51f1fd1cbec32b77c349fb5249 Mon Sep 17 00:00:00 2001 From: jacobbmay <134300709+jacobbmay@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:46:05 -0400 Subject: [PATCH] chore: switch zarf to use s3 backend (#157) * Use s3 backed zarf registry instead of a PV * Replace custom zarf init package with upstream and move csi into standalone package --- bundles/uds-core-swf/uds-bundle.yaml | 25 ++-- config/uds-config.yaml | 37 ++++- packages/init/values/registry-values.yaml | 8 - packages/init/zarf-config.yaml | 17 --- packages/init/zarf.yaml | 139 ------------------ .../nutanix-dynamicfile.yaml | 0 .../values/nutanix-snapshot-values.yaml | 0 .../values/nutanix-storage-values.yaml | 0 packages/nutanix-csi/zarf.yaml | 65 ++++++++ tasks.yaml | 2 +- tasks/create.yaml | 6 +- 11 files changed, 115 insertions(+), 184 deletions(-) delete mode 100644 packages/init/values/registry-values.yaml delete mode 100644 packages/init/zarf-config.yaml delete mode 100644 packages/init/zarf.yaml rename packages/{init => nutanix-csi}/nutanix-dynamicfile.yaml (100%) rename packages/{init => nutanix-csi}/values/nutanix-snapshot-values.yaml (100%) rename packages/{init => nutanix-csi}/values/nutanix-storage-values.yaml (100%) create mode 100644 packages/nutanix-csi/zarf.yaml diff --git a/bundles/uds-core-swf/uds-bundle.yaml b/bundles/uds-core-swf/uds-bundle.yaml index f3339167..506e6d0c 100644 --- a/bundles/uds-core-swf/uds-bundle.yaml +++ b/bundles/uds-core-swf/uds-bundle.yaml @@ -32,26 +32,29 @@ x-gitlab-init-security-context: &gitlab-init-security-context packages: # Zarf init - name: init - path: ../../build - optionalComponents: - - git-server - ref: v0.36.1 + repository: ghcr.io/zarf-dev/packages/init + ref: v0.38.2 overrides: zarf-seed-registry: docker-registry: variables: - - name: REGISTRY_STORAGE_CLASS - path: persistence.storageClass - default: "nutanix-dynamicfile" + - name: REGISTRY_CA_BUNDLE + path: caBundle + default: "" zarf-registry: docker-registry: variables: - - name: REGISTRY_STORAGE_CLASS - path: persistence.storageClass - default: "nutanix-dynamicfile" - - name: DEPLOYMENT_REPLICACOUNT + - name: REGISTRY_CA_BUNDLE + path: caBundle + default: "" + - name: REGISTRY_REPLICA_COUNT path: replicaCount default: 3 + + - name: nutanix-csi + path: ../../build + ref: 0.0.1 + overrides: nutanix-csi-storage: nutanix-csi-storage: variables: diff --git a/config/uds-config.yaml b/config/uds-config.yaml index e24199f6..6c7f3ce4 100644 --- a/config/uds-config.yaml +++ b/config/uds-config.yaml @@ -6,6 +6,12 @@ shared: ADDITIONAL_CA_CHAIN: replace-me-with-additional-ca-chain variables: init: + REGISTRY_HPA_ENABLE: false + REGISTRY_PVC_ENABLED: false + REGISTRY_PVC_SIZE: 128Gi + REGISTRY_PVC_ACCESS_MODE: ReadWriteMany + REGISTRY_HPA_AUTO_SIZE: false + REGISTRY_REPLICA_COUNT: 3 REGISTRY_AFFINITY_CUSTOM: | podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -17,11 +23,32 @@ variables: operator: In values: - docker-registry - topologyKey: kubernetes.io/hostname REGISTRY_HPA_AUTO_SIZE: true - REGISTRY_PVC_ACCESS_MODE: ReadWriteMany - REGISTRY_PVC_ENABLED: true - REGISTRY_PVC_SIZE: 128Gi - REGISTRY_STORAGE_CLASS: "nutanix-dynamicfile" + topologyKey: kubernetes.io/hostname + #Configure zarf registry to use s3 backend. REGISTRY_STORAGE_REDIRECT_DISABLE can be set to false if S3 bucket is accessible and trusted from host that the uds deploy is run from + REGISTRY_EXTRA_ENVS: | + - name: REGISTRY_STORAGE + value: s3 + - name: REGISTRY_STORAGE_S3_REGION + value: us-east-1 + - name: REGISTRY_STORAGE_S3_REGIONENDPOINT + value: nutanix.object.store.hostname + - name: REGISTRY_STORAGE_S3_SECURE + value: "true" + - name: REGISTRY_STORAGE_S3_BUCKET + value: env-zarf-registry-bucket + - name: REGISTRY_STORAGE_S3_ACCESSKEY + value: access-key-id + - name: REGISTRY_STORAGE_S3_SECRETKEY + value: access-key-value + - name: REGISTRY_STORAGE_REDIRECT_DISABLE + value: "true" + REGISTRY_CA_BUNDLE: | + -----BEGIN CERTIFICATE----- + cert contents + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + other cert contents + -----END CERTIFICATE----- PRISM_ENDPOINT: "PRISM element IP address" PRISM_USERNAME: "csi-user-prism-element-user" PRISM_PASSWORD: "csi-user-passoword" diff --git a/packages/init/values/registry-values.yaml b/packages/init/values/registry-values.yaml deleted file mode 100644 index cbb33fb9..00000000 --- a/packages/init/values/registry-values.yaml +++ /dev/null @@ -1,8 +0,0 @@ -persistence: - enabled: ###ZARF_VAR_UPGRADE_PERSISTENCE### - -autoscaling: - enabled: false - -extraEnvVars: - ###ZARF_VAR_UPGRADE_ENV_VARS### diff --git a/packages/init/zarf-config.yaml b/packages/init/zarf-config.yaml deleted file mode 100644 index 05022fd9..00000000 --- a/packages/init/zarf-config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -package: - create: - max_package_size: "1000000000" - # These values are sourced from and updated with https://github.com/defenseunicorns/zarf/blob/main/zarf-config.toml - set: - agent_image_domain: "ghcr.io/" - agent_image: "defenseunicorns/zarf/agent" - - injector_version: "2023-08-02" - injector_amd64_shasum: "91de0768855ee2606a4f85a92bb480ff3a14ca205fd8d05eb397c18e15aa0247" - injector_arm64_shasum: "663df681deea957b0ec53538eab221691a83de8e95d86b8a29008af711934bee" - - registry_image_domain: "" - registry_image: "library/registry" - registry_image_tag: "2.8.3" - - gitea_image: "gitea/gitea:1.21.11-rootless" diff --git a/packages/init/zarf.yaml b/packages/init/zarf.yaml deleted file mode 100644 index 1df788d6..00000000 --- a/packages/init/zarf.yaml +++ /dev/null @@ -1,139 +0,0 @@ -kind: ZarfInitConfig -metadata: - name: init - description: "Nutanix CSI Driver Custom Zarf Init Package" - architecture: amd64 - version: "0.0.1" # This version is not used by zarf, but is used for tracking with the published versions - -variables: - - name: DYNAMIC_FILE_STORE_NAME - description: "Name of Nutanix File Server to use for Dynamic File storageclass. Should match the name value for the file server in Prism." - - name: PRISM_ENDPOINT - description: "IP or hostname of Prism Element." - - name: PRISM_USERNAME - description: "Username of prism user to use for Nutanix CSI driver." - - name: PRISM_PASSWORD - description: "Password for prism user to use for Nutanix CSI driver." - - name: STORAGE_CONTAINER - description: "Name of Nutanix Storage Container for CSI driver to create volumes in." - -components: - # (Optional) Deploys a k3s cluster - - name: k3s - import: - url: oci://ghcr.io/defenseunicorns/packages/init:v0.36.1 - - # This package moves the injector & registries binaries - - name: zarf-injector - required: true - import: - url: oci://ghcr.io/defenseunicorns/packages/init:v0.36.1 - - # Creates the temporary seed-registry - - name: zarf-seed-registry - required: true - import: - url: oci://ghcr.io/defenseunicorns/packages/init:v0.36.1 - charts: - - name: docker-registry - valuesFiles: - - values/registry-values.yaml - # On upgrades ensure we retain the existing PV - actions: - onDeploy: - before: - - description: Set persistence for upgrade seed registry - cmd: ./zarf tools kubectl get pvc zarf-docker-registry -n zarf >/dev/null 2>&1 && echo true || echo false - mute: true - setVariables: - - name: UPGRADE_PERSISTENCE - - description: Set env vars for upgrade seed registry - mute: true - cmd: | - ./zarf tools kubectl get pvc zarf-docker-registry -n zarf >/dev/null 2>&1 && \ - echo "" || \ - echo "- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: \"/var/lib/registry\"" - setVariables: - - name: UPGRADE_ENV_VARS - autoIndent: true - - # Push nutanix csi images to seed-registry - - name: nutanix-csi-images-initial - required: true - description: Push nutanix images to the zarf registry - images: - - registry.k8s.io/sig-storage/snapshot-controller:v8.0.1 - - registry.k8s.io/sig-storage/snapshot-validation-webhook:v8.0.1 - - quay.io/karbon/ntnx-csi:v2.6.10 - - registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 - - registry.k8s.io/sig-storage/csi-provisioner:v5.0.1 - - registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1 - - registry.k8s.io/sig-storage/csi-resizer:v1.11.2 - - registry.k8s.io/sig-storage/livenessprobe:v2.13.1 - - registry1.dso.mil/ironbank/opensource/velero/velero-plugin-for-csi:v0.7.1 - - registry1.dso.mil/ironbank/opensource/velero/velero-plugin-for-aws:v1.10.0 - - - name: nutanix-csi-storage - required: true - charts: - # renovate: datasource=helm - - name: nutanix-csi-storage - url: https://github.com/defenseunicorns/nutanix-helm.git # fork containing fix for imagepullsecrets needed for pods to pull images from zarf registry - version: v2.6.10-modified - gitPath: charts/nutanix-csi-storage - namespace: ntnx-system - valuesFiles: - - values/nutanix-storage-values.yaml - actions: - onDeploy: - before: - - description: Delete Storage Classes - cmd: ./zarf tools kubectl delete sc nutanix-volume --ignore-not-found=true - - - name: nutanix-dynamicfile-manifests - required: true - manifests: - - name: nutanix-dynamicfile-manifests - namespace: ntnx-system - files: - - nutanix-dynamicfile.yaml - actions: - onDeploy: - before: - - description: Delete Storage Classes - cmd: ./zarf tools kubectl delete sc nutanix-dynamicfile --ignore-not-found=true - - # Creates the permanent registry - - name: zarf-registry - required: true - import: - url: oci://ghcr.io/defenseunicorns/packages/init:v0.36.1 - - # Push nutanix csi (and registry) images to permanent registry - - name: nutanix-csi-images - required: true - description: Push nutanix csi images to the zarf registry - images: - - registry.k8s.io/sig-storage/snapshot-controller:v8.0.1 - - registry.k8s.io/sig-storage/snapshot-validation-webhook:v8.0.1 - - quay.io/karbon/ntnx-csi:v2.6.10 - - registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 - - registry.k8s.io/sig-storage/csi-provisioner:v5.0.1 - - registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1 - - registry.k8s.io/sig-storage/csi-resizer:v1.11.2 - - registry.k8s.io/sig-storage/livenessprobe:v2.13.1 - - registry1.dso.mil/ironbank/opensource/velero/velero-plugin-for-csi:v0.7.1 - - registry1.dso.mil/ironbank/opensource/velero/velero-plugin-for-aws:v1.10.0 - - "###ZARF_PKG_TMPL_REGISTRY_IMAGE_DOMAIN######ZARF_PKG_TMPL_REGISTRY_IMAGE###:###ZARF_PKG_TMPL_REGISTRY_IMAGE_TAG###" - - # Creates the pod+git mutating webhook - - name: zarf-agent - required: true - import: - url: oci://ghcr.io/defenseunicorns/packages/init:v0.36.1 - - # (Optional) Adds a git server to the cluster - - name: git-server - import: - url: oci://ghcr.io/defenseunicorns/packages/init:v0.36.1 diff --git a/packages/init/nutanix-dynamicfile.yaml b/packages/nutanix-csi/nutanix-dynamicfile.yaml similarity index 100% rename from packages/init/nutanix-dynamicfile.yaml rename to packages/nutanix-csi/nutanix-dynamicfile.yaml diff --git a/packages/init/values/nutanix-snapshot-values.yaml b/packages/nutanix-csi/values/nutanix-snapshot-values.yaml similarity index 100% rename from packages/init/values/nutanix-snapshot-values.yaml rename to packages/nutanix-csi/values/nutanix-snapshot-values.yaml diff --git a/packages/init/values/nutanix-storage-values.yaml b/packages/nutanix-csi/values/nutanix-storage-values.yaml similarity index 100% rename from packages/init/values/nutanix-storage-values.yaml rename to packages/nutanix-csi/values/nutanix-storage-values.yaml diff --git a/packages/nutanix-csi/zarf.yaml b/packages/nutanix-csi/zarf.yaml new file mode 100644 index 00000000..f2097865 --- /dev/null +++ b/packages/nutanix-csi/zarf.yaml @@ -0,0 +1,65 @@ +kind: ZarfPackageConfig +metadata: + name: nutanix-csi + description: "Nutanix CSI Driver Package" + architecture: amd64 + version: "0.0.1" # This version is not used by zarf, but is used for tracking with the published versions + +variables: + - name: DYNAMIC_FILE_STORE_NAME + description: "Name of Nutanix File Server to use for Dynamic File storageclass. Should match the name value for the file server in Prism." + - name: PRISM_ENDPOINT + description: "IP or hostname of Prism Element." + - name: PRISM_USERNAME + description: "Username of prism user to use for Nutanix CSI driver." + - name: PRISM_PASSWORD + description: "Password for prism user to use for Nutanix CSI driver." + - name: STORAGE_CONTAINER + description: "Name of Nutanix Storage Container for CSI driver to create volumes in." + +components: + # Push nutanix csi images to seed-registry + - name: nutanix-csi-images-initial + required: true + description: Push nutanix images to the zarf registry + images: + - registry.k8s.io/sig-storage/snapshot-controller:v8.0.1 + - registry.k8s.io/sig-storage/snapshot-validation-webhook:v8.0.1 + - quay.io/karbon/ntnx-csi:v2.6.10 + - registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1 + - registry.k8s.io/sig-storage/csi-provisioner:v5.0.1 + - registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1 + - registry.k8s.io/sig-storage/csi-resizer:v1.11.2 + - registry.k8s.io/sig-storage/livenessprobe:v2.13.1 + - registry1.dso.mil/ironbank/opensource/velero/velero-plugin-for-csi:v0.7.1 + - registry1.dso.mil/ironbank/opensource/velero/velero-plugin-for-aws:v1.10.0 + + - name: nutanix-csi-storage + required: true + charts: + # renovate: datasource=helm + - name: nutanix-csi-storage + url: https://github.com/defenseunicorns/nutanix-helm.git # fork containing fix for imagepullsecrets needed for pods to pull images from zarf registry + version: v2.6.10-modified + gitPath: charts/nutanix-csi-storage + namespace: ntnx-system + valuesFiles: + - values/nutanix-storage-values.yaml + actions: + onDeploy: + before: + - description: Delete Storage Classes + cmd: ./zarf tools kubectl delete sc nutanix-volume --ignore-not-found=true + + - name: nutanix-dynamicfile-manifests + required: true + manifests: + - name: nutanix-dynamicfile-manifests + namespace: ntnx-system + files: + - nutanix-dynamicfile.yaml + actions: + onDeploy: + before: + - description: Delete Storage Classes + cmd: ./zarf tools kubectl delete sc nutanix-dynamicfile --ignore-not-found=true diff --git a/tasks.yaml b/tasks.yaml index f639c448..b00dc3a0 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -13,7 +13,7 @@ tasks: - task: create:keycloak-config-wrapper-package - task: create:object-store-packages - task: create:additional-manifests-package - - task: create:init-package + - task: create:nutanix-csi-package - task: create:trust-manager-package - task: create:trust-bundles-package - task: create:bundle diff --git a/tasks/create.yaml b/tasks/create.yaml index 81d8dd6f..00400883 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -41,10 +41,10 @@ tasks: actions: - cmd: ./uds zarf package create ./packages/additional-manifests --confirm --no-progress --architecture=${ARCH} --skip-sbom --output ./build - - name: init-package - description: Create init package with Nutanix CSI driver. + - name: nutanix-csi-package + description: Create Nutanix CSI driver package. actions: - - cmd: ZARF_CONFIG=./packages/init/zarf-config.yaml ./uds zarf package create ./packages/init --set AGENT_IMAGE_TAG=$(uds zarf version) --confirm --no-progress --architecture=${ARCH} --skip-sbom --output ./build + - cmd: ./uds zarf package create ./packages/nutanix-csi --confirm --no-progress --architecture=${ARCH} --skip-sbom --output ./build - name: trust-manager-package description: Create trust-manager package.