Add the ability to specify Zarf variables as filepaths

docs/3-create-a-zarf-package/4-zarf-schema.md

@@ -2202,6 +2202,28 @@ Must be one of:
 </blockquote>
 </details>
 
+<details>
+<summary>
+<strong> <a name="components_items_actions_onCreate_before_items_setVariables_items_type"></a>type</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+**Description:** Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB)
+
+|          |                    |
+| -------- | ------------------ |
+| **Type** | `enum (of string)` |
+
+:::note
+Must be one of:
+* "raw"
+* "file"
+:::
+
+</blockquote>
+</details>
+
 </blockquote>
 </details>
 
@@ -2811,6 +2833,28 @@ Must be one of:
 </blockquote>
 </details>
 
+<details>
+<summary>
+<strong> <a name="variables_items_type"></a>type</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+**Description:** Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB)
+
+|          |                    |
+| -------- | ------------------ |
+| **Type** | `enum (of string)` |
+
+:::note
+Must be one of:
+* "raw"
+* "file"
+:::
+
+</blockquote>
+</details>
+
 </blockquote>
 </details>
 

examples/variables/README.md

@@ -1,3 +1,4 @@
+import Properties from '@site/src/components/SchemaItemProperties';
 import ExampleYAML from "@site/src/components/ExampleYAML";
 
 # Variables
@@ -23,7 +24,7 @@ To use variables and constants at deploy time you need to have two things:
 1. a manifest that you want to template a value in
 2. a defined variable in the `zarf.yaml` file from `variables` or `setVariable`
 
-The manifest should have your desired variable name in ALL CAPS prefixed with `###ZARF_VAR`  for `variables` or prefixed with `###ZARF_CONST` for `constants` and suffixed with `###`.  For example in a configmap that took a variable named `DATABASE_USERNAME` you would provide the following:
+The manifest should have your desired variable name in ALL CAPS prefixed with `###ZARF_VAR` for `variables` or prefixed with `###ZARF_CONST` for `constants` and suffixed with `###`.  For example in a configmap that took a variable named `DATABASE_USERNAME` you would provide the following:
 
 ```yaml
 apiVersion: v1
@@ -34,7 +35,7 @@ data:
   username: ###ZARF_VAR_DATABASE_USERNAME###
 ```
 
-In the `zarf.yaml`, you would need to define the variable in the `variables` section or as output from an action with `setVariable` with the same `name` as above. Or for a constant you would use the `constants` section.  For the same example as above, I would have the following for a variable defined by the deploy user:
+In the `zarf.yaml`, you would need to define the variable in the `variables` section or as output from an action with `setVariable` with the same `name` as above. Or for a constant you would use the `constants` section.  For the same example as above, you would have the following for a variable defined by the deploy user:
 
 ```yaml
 variables:
@@ -55,6 +56,20 @@ components:
               - name: DATABASE_USERNAME
 ```
 
+Zarf `variables` can also have additional fields that describe how Zarf will handle them which are described below:
+
+<Properties item="ZarfPackageVariable" />
+
+:::note
+
+The fields `default`, `description` and `prompt` are not available on `setVariables` since they always take the standard output of an action command and will not be interacted with directly by a deploy user.
+
+:::
+
+Zarf `constants` are similar but have fewer options as they are static by the time `zarf package deploy` is run:
+
+<Properties item="ZarfPackageConstant" />
+
 :::note
 
 All names must match the regex pattern `^[A-Z0-9_]+$` [Test](https://regex101.com/r/BG5ZqW/1)).
@@ -63,7 +78,7 @@ All names must match the regex pattern `^[A-Z0-9_]+$` [Test](https://regex101.co
 
 :::tip
 
-When not specifying `default`, `prompt`, `sensitive`, or `indent` Zarf will default to `default: ""`, `prompt: false`, `sensitive: false` and `indent: 0`
+When not specifying `default`, `prompt`, `sensitive`, `autoIndent`, or `type` Zarf will default to `default: ""`, `prompt: false`, `sensitive: false`, `autoIndent: false`, and `type: "raw"`
 
 :::
 

examples/variables/nginx-configmap.yaml

@@ -24,6 +24,7 @@ data:
         <pre>
           ###ZARF_VAR_MODIFIED_TERRAFORM###
         </pre>
+        <pre>File SHASUM: ###ZARF_VAR_MODIFIED_TERRAFORM_SHASUM###</pre>
         ###ZARF_VAR_OPTIONAL_FOOTER###
       </body>
     </html>

examples/variables/zarf.yaml

@@ -35,10 +35,16 @@ variables:
   - name: AWS_REGION
     default: us-east-1
     sensitive: true
+  # MODIFIED_TERRAFORM sets a filepath for a terraform file to be used as the contents of a template
+  - name: MODIFIED_TERRAFORM
+    default: modified-terraform.tf
+    autoIndent: true
+    sensitive: true
+    type: file
 
 components:
   # The following component templates the provided .tf file with the defined AWS_REGION
-  # NOTE: this component does not actually execute this file in this example (see examples/terraform)
+  # NOTE: this component does not actually execute this file in this example
   - name: variables-with-terraform
     description: Change a value in a regular file with a Zarf variable. Set AWS_REGION variable to modify the file.
     required: true
@@ -48,14 +54,13 @@ components:
     actions:
       onDeploy:
         after:
-          # This command `cat`s the modified terraform file on deploy for use later on (see examples/component-actions for more)
-          - cmd: cat modified-terraform.tf
-            # `mute` is set to exclude the command output from being shown (note this will include AWS_REGION which we marked sensitive above)
+          # This command uses Zarf to return the SHASUM of the terraform file (`type: file` variables will return the filepath instead of the contents when used in actions)
+          - cmd: ./zarf prepare sha256sum ${ZARF_VAR_MODIFIED_TERRAFORM}
+            # `mute` is set to exclude the command output from being shown (since we are treating it as sensitive below)
             mute: true
             setVariables:
-              - name: MODIFIED_TERRAFORM
-                autoIndent: true
-                # `sensitive` is set to exclude the command output from the logs (note this will include AWS_REGION which we marked sensitive above)
+              - name: MODIFIED_TERRAFORM_SHASUM
+                # `sensitive` is set to exclude the command output from the logs
                 sensitive: true
 
   # The following component deploys nginx to the cluster using the defined variables

src/internal/packager/template/template.go

@@ -139,6 +139,7 @@ func (values *Values) GetVariables(component types.ZarfComponent) (map[string]*u
 			Value:      variable.Value,
 			Sensitive:  variable.Sensitive,
 			AutoIndent: variable.AutoIndent,
+			Type:       variable.Type,
 		}
 	}
 

src/pkg/packager/actions.go

@@ -111,7 +111,7 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio
 
 			// If an output variable is defined, set it.
 			for _, v := range action.SetVariables {
-				p.setVariableInConfig(v.Name, out, v.Sensitive, v.AutoIndent)
+				p.setVariableInConfig(v.Name, out, v.Sensitive, v.AutoIndent, v.Type)
 			}
 
 			// If the action has a wait, change the spinner message to reflect that on success.

src/pkg/packager/variables.go

@@ -76,7 +76,7 @@ func (p *Packager) setVariableMapInConfig() error {
 	// Ensure uppercase keys
 	setVariableValues := helpers.TransformMapKeys(p.cfg.DeployOpts.SetVariables, strings.ToUpper)
 	for name, value := range setVariableValues {
-		p.setVariableInConfig(name, value, false, false)
+		p.setVariableInConfig(name, value, false, false, "")
 	}
 
 	for _, variable := range p.cfg.Pkg.Variables {
@@ -86,11 +86,12 @@ func (p *Packager) setVariableMapInConfig() error {
 		if present {
 			p.cfg.SetVariableMap[variable.Name].Sensitive = variable.Sensitive
 			p.cfg.SetVariableMap[variable.Name].AutoIndent = variable.AutoIndent
+			p.cfg.SetVariableMap[variable.Name].Type = variable.Type
 			continue
 		}
 
 		// First set default (may be overridden by prompt)
-		p.setVariableInConfig(variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent)
+		p.setVariableInConfig(variable.Name, variable.Default, variable.Sensitive, variable.AutoIndent, variable.Type)
 
 		// Variable is set to prompt the user
 		if variable.Prompt && !config.CommonOptions.Confirm {
@@ -101,19 +102,20 @@ func (p *Packager) setVariableMapInConfig() error {
 				return err
 			}
 
-			p.setVariableInConfig(variable.Name, val, variable.Sensitive, variable.AutoIndent)
+			p.setVariableInConfig(variable.Name, val, variable.Sensitive, variable.AutoIndent, variable.Type)
 		}
 	}
 
 	return nil
 }
 
-func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoIndent bool) {
+func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoIndent bool, varType types.VariableType) {
 	p.cfg.SetVariableMap[name] = &types.ZarfSetVariable{
 		Name:       name,
 		Value:      value,
 		Sensitive:  sensitive,
 		AutoIndent: autoIndent,
+		Type:       varType,
 	}
 }
 

src/pkg/utils/io.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/utils/helpers"
+	"github.com/defenseunicorns/zarf/src/types"
 	"github.com/otiai10/copy"
 )
 
@@ -54,6 +55,7 @@ func GetCryptoHashFromFile(path string, hashName crypto.Hash) (string, error) {
 type TextTemplate struct {
 	Sensitive  bool
 	AutoIndent bool
+	Type       types.VariableType
 	Value      string
 }
 
@@ -152,6 +154,14 @@ func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, depreca
 	regexTemplateLine := regexp.MustCompile(fmt.Sprintf("(?P<preTemplate>.*?)(?P<template>%s)(?P<postTemplate>.*)", templateRegex))
 
 	fileScanner := bufio.NewScanner(textFile)
+
+	// Set the buffer to 1 MiB to handle long lines (i.e. base64 text in a secret)
+	// 1 MiB is around the documented maximum size for secrets and configmaps
+	const maxCapacity = 1024 * 1024
+	buf := make([]byte, maxCapacity)
+	fileScanner.Buffer(buf, maxCapacity)
+
+	// Set the scanner to split on new lines
 	fileScanner.Split(bufio.ScanLines)
 
 	text := ""
@@ -183,6 +193,24 @@ func ReplaceTextTemplate(path string, mappings map[string]*TextTemplate, depreca
 			if template != nil {
 				value = template.Value
 
+				// Check if the value is a file type and load the value contents from the file
+				if template.Type == types.FileVariableType {
+					if isText, err := IsTextFile(value); err != nil || !isText {
+						message.Warnf("Refusing to load a non-text file for templating %s", templateKey)
+						line = matches[regexTemplateLine.SubexpIndex("postTemplate")]
+						continue
+					}
+
+					contents, err := os.ReadFile(value)
+					if err != nil {
+						message.Warnf("Unable to read file for templating - skipping: %s", err.Error())
+						line = matches[regexTemplateLine.SubexpIndex("postTemplate")]
+						continue
+					}
+
+					value = string(contents)
+				}
+
 				// Check if the value is autoIndented and add the correct spacing
 				if template.AutoIndent {
 					indent := fmt.Sprintf("\n%s", strings.Repeat(" ", len(preTemplate)))

src/test/e2e/24_variables_test.go

@@ -57,6 +57,8 @@ func TestVariables(t *testing.T) {
 	require.Contains(t, string(kubectlOut), "Defense Unicorns")
 	// AWS_REGION should have been templated and also templated into this config map
 	require.Contains(t, string(kubectlOut), "unicorn-land")
+	// MODIFIED_TERRAFORM_SHASUM should have been templated
+	require.Contains(t, string(kubectlOut), "0ce06459b27122d42703ba7bd50c65d6aa4d23b4d2344e1c93aea018fdd64219")
 
 	// Remove the variables example
 	stdOut, stdErr, err = e2e.Zarf("package", "remove", path, "--confirm")

src/test/e2e/25_helm_test.go

@@ -70,12 +70,13 @@ func testHelmEscaping(t *testing.T) {
 	stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--confirm")
 	require.NoError(t, err, stdOut, stdErr)
 
-	// Verify the configmap was deployed and escaped.
+	// Verify the configmap was deployed, escaped, and contains all of its data
 	kubectlOut, _ := exec.Command("kubectl", "describe", "cm", "dont-template-me").Output()
 	require.Contains(t, string(kubectlOut), `alert: OOMKilled {{ "{{ \"random.Values\" }}" }}`)
 	require.Contains(t, string(kubectlOut), "backtick1: \"content with backticks `some random things`\"")
 	require.Contains(t, string(kubectlOut), "backtick2: \"nested templating with backticks {{` random.Values `}}\"")
 	require.Contains(t, string(kubectlOut), `description: Pod {{$labels.pod}} in {{$labels.namespace}} got OOMKilled`)
+	require.Contains(t, string(kubectlOut), `TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNlY3RldHVyIG`)
 
 	// Remove the package.
 	stdOut, stdErr, err = e2e.Zarf("package", "remove", "evil-templates", "--confirm")

src/types/component.go

@@ -169,9 +169,10 @@ type ZarfComponentAction struct {
 
 // ZarfComponentActionSetVariable represents a variable that is to be set via an action
 type ZarfComponentActionSetVariable struct {
-	Name       string `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"`
-	Sensitive  bool   `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"`
-	AutoIndent bool   `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."`
+	Name       string       `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"`
+	Sensitive  bool         `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"`
+	AutoIndent bool         `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."`
+	Type       VariableType `json:"type,omitempty" jsonschema:"description=Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB),enum=raw,enum=file"`
 }
 
 // ZarfComponentActionWait specifies a condition to wait for before continuing

src/types/package.go

@@ -47,12 +47,13 @@ type ZarfBuildData struct {
 
 // ZarfPackageVariable are variables that can be used to dynamically template K8s resources.
 type ZarfPackageVariable struct {
-	Name        string `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"`
-	Description string `json:"description,omitempty" jsonschema:"description=A description of the variable to be used when prompting the user a value"`
-	Default     string `json:"default,omitempty" jsonschema:"description=The default value to use for the variable"`
-	Prompt      bool   `json:"prompt,omitempty" jsonschema:"description=Whether to prompt the user for input for this variable"`
-	Sensitive   bool   `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"`
-	AutoIndent  bool   `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."`
+	Name        string       `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"`
+	Description string       `json:"description,omitempty" jsonschema:"description=A description of the variable to be used when prompting the user a value"`
+	Default     string       `json:"default,omitempty" jsonschema:"description=The default value to use for the variable"`
+	Prompt      bool         `json:"prompt,omitempty" jsonschema:"description=Whether to prompt the user for input for this variable"`
+	Sensitive   bool         `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"`
+	AutoIndent  bool         `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."`
+	Type        VariableType `json:"type,omitempty" jsonschema:"description=Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB),enum=raw,enum=file"`
 }
 
 // ZarfPackageConstant are constants that can be used to dynamically template K8s resources.

src/types/runtime.go

@@ -13,8 +13,13 @@ const (
 	ManifestsFolder      = "manifests"
 	DataInjectionsFolder = "data"
 	ValuesFolder         = "values"
+
+	RawVariableType  VariableType = "raw"
+	FileVariableType VariableType = "file"
 )
 
+type VariableType string
+
 // ZarfCommonOptions tracks the user-defined preferences used across commands.
 type ZarfCommonOptions struct {
 	Confirm        bool   `json:"confirm" jsonschema:"description=Verify that Zarf should perform an action"`
@@ -86,10 +91,11 @@ type ZarfPartialPackageData struct {
 
 // ZarfSetVariable tracks internal variables that have been set during this run of Zarf
 type ZarfSetVariable struct {
-	Name       string `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"`
-	Sensitive  bool   `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"`
-	AutoIndent bool   `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."`
-	Value      string `json:"value" jsonschema:"description=The value the variable is currently set with"`
+	Name       string       `json:"name" jsonschema:"description=The name to be used for the variable,pattern=^[A-Z0-9_]+$"`
+	Sensitive  bool         `json:"sensitive,omitempty" jsonschema:"description=Whether to mark this variable as sensitive to not print it in the Zarf log"`
+	AutoIndent bool         `json:"autoIndent,omitempty" jsonschema:"description=Whether to automatically indent the variable's value (if multiline) when templating. Based on the number of chars before the start of ###ZARF_VAR_."`
+	Value      string       `json:"value" jsonschema:"description=The value the variable is currently set with"`
+	Type       VariableType `json:"type,omitempty" jsonschema:"description=Changes the handling of a variable to load contents differently (i.e. from a file rather than as a raw variable - templated files should be kept below 1 MiB),enum=raw,enum=file"`
 }
 
 // ConnectString contains information about a connection made with Zarf connect.