Add support for setting dependent mod variable values using an spvars…

… or by setting and Args property in the mod Require block. Closes #2076. Closes #2077

cmd/mod.go

@@ -226,9 +226,8 @@ func runModInitCmd(cmd *cobra.Command, args []string) {
 		fmt.Println("Working folder already contains a mod definition file")
 		return
 	}
-	mod, err := modconfig.CreateDefaultMod(workspacePath)
-	utils.FailOnError(err)
-	err = mod.Save()
+	mod := modconfig.CreateDefaultMod(workspacePath)
+	err := mod.Save()
 	utils.FailOnError(err)
 	fmt.Printf("Created mod definition file '%s'\n", filepaths.ModFilePath(workspacePath))
 }

cmd/query.go

@@ -161,7 +161,7 @@ func loadWorkspacePromptingForVariables(ctx context.Context) (*workspace.Workspa
 	// so we have missing variables - prompt for them
 	// first hide spinner if it is there
 	statushooks.Done(ctx)
-	if err := interactive.PromptForMissingVariables(ctx, missingVariablesError.MissingVariables); err != nil {
+	if err := interactive.PromptForMissingVariables(ctx, missingVariablesError.MissingVariables, workspacePath); err != nil {
 		log.Printf("[TRACE] Interactive variables prompting returned error %v", err)
 		return nil, err
 	}

interactive/variables.go

@@ -11,24 +11,31 @@ import (
 	"github.com/turbot/steampipe/steampipeconfig/modconfig"
 )
 
-func PromptForMissingVariables(ctx context.Context, missingVariables []*modconfig.Variable) error {
+func PromptForMissingVariables(ctx context.Context, missingVariables []*modconfig.Variable, workspacePath string) error {
 	fmt.Println()
 	fmt.Println("Variables defined with no value set.")
 	for _, v := range missingVariables {
-		r, err := promptForVariable(ctx, v.ShortName, v.Description)
+		variableName := v.ShortName
+		variableDisplayName := fmt.Sprintf("var.%s", v.ShortName)
+		// if this variable is NOT part of the workspace mod, add the mod name to the variable name
+		if v.Mod.ModPath != workspacePath {
+			variableDisplayName = fmt.Sprintf("%s.var.%s", v.ModName, v.ShortName)
+			variableName = fmt.Sprintf("%s.%s", v.ModName, v.ShortName)
+		}
+		r, err := promptForVariable(ctx, variableDisplayName, v.Description)
 		if err != nil {
 			return err
 		}
-		addInteractiveVariableToViper(v.ShortName, r)
+		addInteractiveVariableToViper(variableName, r)
 	}
 	return nil
 }
 
 func promptForVariable(ctx context.Context, name, description string) (string, error) {
 	uiInput := &inputvars.UIInput{}
 	rawValue, err := uiInput.Input(ctx, &terraform.InputOpts{
-		Id:          fmt.Sprintf("var.%s", name),
-		Query:       fmt.Sprintf("var.%s", name),
+		Id:          name,
+		Query:       name,
 		Description: description,
 	})
 

modinstaller/mod_installer.go

@@ -451,7 +451,8 @@ func (i *ModInstaller) setModDependencyPath(mod *modconfig.Mod, modPath string)
 func (i *ModInstaller) loadModfile(modPath string, createDefault bool) (*modconfig.Mod, error) {
 	if !parse.ModfileExists(modPath) {
 		if createDefault {
-			return modconfig.CreateDefaultMod(i.workspacePath)
+			mod := modconfig.CreateDefaultMod(i.workspacePath)
+			return mod, nil
 		}
 		return nil, nil
 	}

steampipeconfig/inputvars/collect_variables.go

@@ -3,16 +3,19 @@ package inputvars
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strings"
 
-	"github.com/spf13/viper"
-
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/terraform/tfdiags"
+	"github.com/spf13/viper"
 	"github.com/turbot/steampipe/constants"
 	"github.com/turbot/steampipe/filepaths"
+	"github.com/turbot/steampipe/steampipeconfig/modconfig"
 	"github.com/turbot/steampipe/steampipeconfig/modconfig/var_config"
+	"github.com/turbot/steampipe/steampipeconfig/parse"
+	"github.com/turbot/steampipe/utils"
 )
 
 // CollectVariableValues inspects the various places that configuration input variable
@@ -22,12 +25,12 @@ import (
 // This method returns diagnostics relating to the collection of the values,
 // but the values themselves may produce additional diagnostics when finally
 // parsed.
-func CollectVariableValues(workspacePath string, variableFileArgs []string, variablesArgs []string) (map[string]UnparsedVariableValue, tfdiags.Diagnostics) {
-	var diags tfdiags.Diagnostics
+func CollectVariableValues(workspacePath string, variableFileArgs []string, variablesArgs []string) (map[string]UnparsedVariableValue, error) {
 	ret := map[string]UnparsedVariableValue{}
 
-	// First we'll deal with environment variables, since they have the lowest
-	// precedence.
+	// First we'll deal with environment variables
+	// since they have the lowest precedence.
+	// (apart from values in the mod Require proeprty, which are handled separately later)
 	{
 		env := os.Environ()
 		for _, raw := range env {
@@ -59,8 +62,11 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 	// ending in .auto.spvars.
 	defaultVarsPath := filepaths.DefaultVarsFilePath(workspacePath)
 	if _, err := os.Stat(defaultVarsPath); err == nil {
-		moreDiags := addVarsFromFile(defaultVarsPath, ValueFromAutoFile, ret)
-		diags = diags.Append(moreDiags)
+		diags := addVarsFromFile(defaultVarsPath, ValueFromAutoFile, ret)
+		if diags.HasErrors() {
+			return nil, utils.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", defaultVarsPath), diags)
+		}
+
 	}
 
 	if infos, err := os.ReadDir("."); err == nil {
@@ -70,17 +76,24 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 			if !isAutoVarFile(name) {
 				continue
 			}
-			moreDiags := addVarsFromFile(name, ValueFromAutoFile, ret)
-			diags = diags.Append(moreDiags)
+			diags := addVarsFromFile(name, ValueFromAutoFile, ret)
+			if diags.HasErrors() {
+				return nil, utils.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", name), diags)
+			}
+
 		}
 	}
 
 	// Finally we process values given explicitly on the command line, either
 	// as individual literal settings or as additional files to read.
 	for _, fileArg := range variableFileArgs {
-		moreDiags := addVarsFromFile(fileArg, ValueFromNamedFile, ret)
-		diags = diags.Append(moreDiags)
+		diags := addVarsFromFile(fileArg, ValueFromNamedFile, ret)
+		if diags.HasErrors() {
+			return nil, utils.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", fileArg), diags)
+		}
 	}
+
+	var diags tfdiags.Diagnostics
 	for _, variableArg := range variablesArgs {
 		// Value should be in the form "name=value", where value is a
 		// raw string whose interpretation will depend on the variable's
@@ -105,6 +118,10 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 		}
 	}
 
+	if diags.HasErrors() {
+		return nil, utils.DiagsToError(fmt.Sprintf("failed to evaluate var args:"), diags)
+	}
+
 	// check viper for any interactively added variables
 	if varMap := viper.GetStringMap(constants.ConfigInteractiveVariables); varMap != nil {
 		for name, rawVal := range varMap {
@@ -120,7 +137,48 @@ func CollectVariableValues(workspacePath string, variableFileArgs []string, vari
 
 	// now map any variable names of form <modname>.<variablename> to <modname>.var.<varname>
 	ret = transformVarNames(ret)
-	return ret, diags
+	return ret, nil
+}
+
+func CollectVariableValuesFromModRequire(mod *modconfig.Mod, runCtx *parse.RunContext) (InputValues, error) {
+	res := make(InputValues)
+	if mod.Require != nil {
+		for _, depModConstraint := range mod.Require.Mods {
+			// find the short name for this mod
+			depMod, ok := runCtx.LoadedDependencyMods[depModConstraint.Name]
+			if !ok {
+				return nil, fmt.Errorf("depency mod %s is not loaded", depMod.Name())
+			}
+
+			if args := depModConstraint.Args; args != nil {
+				for varName, varVal := range args {
+					varFullName := fmt.Sprintf("%s.var.%s", depMod.ShortName, varName)
+
+					sourceRange := tfdiags.SourceRange{
+						Filename: mod.Require.DeclRange.Filename,
+						Start: tfdiags.SourcePos{
+							Line:   mod.Require.DeclRange.Start.Line,
+							Column: mod.Require.DeclRange.Start.Column,
+							Byte:   mod.Require.DeclRange.Start.Byte,
+						},
+						End: tfdiags.SourcePos{
+							Line:   mod.Require.DeclRange.End.Line,
+							Column: mod.Require.DeclRange.End.Column,
+							Byte:   mod.Require.DeclRange.End.Byte,
+						},
+					}
+
+					res[varFullName] = &InputValue{
+						Value:       varVal,
+						SourceType:  ValueFromModFile,
+						SourceRange: sourceRange,
+					}
+
+				}
+			}
+		}
+	}
+	return res, nil
 }
 
 // map any variable names of form <modname>.<variablename> to <modname>.var.<varname>
@@ -156,10 +214,14 @@ func addVarsFromFile(filename string, sourceType ValueSourceType, to map[string]
 		return diags
 	}
 
-	var f *hcl.File
+	// replace syntax `<modname>.<varname>=<var_value>` with `___steampipe_<modname>_<varname>=<var_value>
+	sanitisedSrc, depVarAliases := sanitiseVariableNames(src)
 
+	var f *hcl.File
 	var hclDiags hcl.Diagnostics
-	f, hclDiags = hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1})
+
+	// attempt to parse the config
+	f, hclDiags = hclsyntax.ParseConfig(sanitisedSrc, filename, hcl.Pos{Line: 1, Column: 1})
 	diags = diags.Append(hclDiags)
 	if f == nil || f.Body == nil {
 		return diags
@@ -200,6 +262,10 @@ func addVarsFromFile(filename string, sourceType ValueSourceType, to map[string]
 	diags = diags.Append(hclDiags)
 
 	for name, attr := range attrs {
+		// check for aliases
+		if alias, ok := depVarAliases[name]; ok {
+			name = alias
+		}
 		to[name] = unparsedVariableValueExpression{
 			expr:       attr.Expr,
 			sourceType: sourceType,
@@ -208,6 +274,34 @@ func addVarsFromFile(filename string, sourceType ValueSourceType, to map[string]
 	return diags
 }
 
+func sanitiseVariableNames(src []byte) ([]byte, map[string]string) {
+	// replace syntax `<modname>.<varname>=<var_value>` with `____steampipe_mod_<modname>_<varname>____=<var_value>
+
+	lines := strings.Split(string(src), "\n")
+	// make map of varname aliases
+	var depVarAliases = make(map[string]string)
+
+	for i, line := range lines {
+
+		r := regexp.MustCompile(`^ ?(([a-z0-9\-_]+)\.([a-z0-9\-_]+)) ?=`)
+		captureGroups := r.FindStringSubmatch(line)
+		if captureGroups != nil && len(captureGroups) == 4 {
+			fullVarName := captureGroups[1]
+			mod := captureGroups[2]
+			varName := captureGroups[3]
+
+			aliasedName := fmt.Sprintf("____steampipe_mod_%s_variable_%s____", mod, varName)
+			depVarAliases[aliasedName] = fullVarName
+			lines[i] = strings.Replace(line, fullVarName, aliasedName, 1)
+
+		}
+	}
+
+	// now try again
+	src = []byte(strings.Join(lines, "\n"))
+	return src, depVarAliases
+}
+
 // unparsedVariableValueLiteral is a UnparsedVariableValue
 // implementation that was actually already parsed (!). This is
 // intended to deal with expressions inside "tfvars" files.

steampipeconfig/inputvars/input_values.go

@@ -56,6 +56,9 @@ const (
 	// ValueFromInput indicates that the value was provided at an interactive
 	// input prompt.
 	ValueFromInput ValueSourceType = 'I'
+
+	// ValueFromModFile indicates that the value was provided in the 'Require' section of a mod file
+	ValueFromModFile ValueSourceType = 'M'
 )
 
 func (v *InputValue) GoString() string {
@@ -213,6 +216,14 @@ func (vv InputValues) Identical(other InputValues) bool {
 	return true
 }
 
+func (vv InputValues) DefaultTo(other InputValues) {
+	for k, otherVal := range other {
+		if val, ok := vv[k]; !ok || !val.Value.IsKnown() {
+			vv[k] = otherVal
+		}
+	}
+}
+
 // CheckInputVariables ensures that variable values supplied at the UI conform
 // to their corresponding declarations in configuration.
 //

steampipeconfig/inputvars/unparsed_value.go

@@ -42,7 +42,7 @@ type UnparsedVariableValue interface {
 // InputValues may be incomplete but will include the subset of variables
 // that were successfully processed, allowing for careful analysis of the
 // partial result.
-func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, variablesMap map[string]*modconfig.Variable, validate bool) (InputValues, tfdiags.Diagnostics) {
+func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, variablesMap map[string]*modconfig.Variable, depModVarValues InputValues, validate bool) (InputValues, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 	ret := make(InputValues, len(inputValuesUnparsed))
 
@@ -118,6 +118,10 @@ func ParseVariableValues(inputValuesUnparsed map[string]UnparsedVariableValue, v
 		})
 	}
 
+	// depModVarValues are values of dependency mod variables which are set in the mod file.
+	// default the inputVariables to these values (last resourt)
+	ret.DefaultTo(depModVarValues)
+
 	// By this point we should've gathered all of the required variables
 	// from one of the many possible sources.
 	// We'll now populate any we haven't gathered as their defaults and fail if any of the

steampipeconfig/load_config_test.go

@@ -243,12 +243,12 @@ var testCasesLoadConfig = map[string]loadConfigTest{
 	},
 	"single_connection_with_default_options_and_workspace_invalid_options_block": { // fixed
 		steampipeDir: "testdata/connection_config/single_connection_with_default_options",
-		workspaceDir: "testdata/workspaces/invalid_options_block",
+		workspaceDir: "testdata/load_config_test/invalid_options_block",
 		expected:     "ERROR",
 	},
 	"single_connection_with_default_options_and_workspace_search_path_prefix": { // fixed
 		steampipeDir: "testdata/connection_config/single_connection_with_default_options",
-		workspaceDir: "testdata/workspaces/search_path_prefix",
+		workspaceDir: "testdata/load_config_test/search_path_prefix",
 		expected: &SteampipeConfig{
 			Connections: map[string]*modconfig.Connection{
 				"a": {
@@ -317,7 +317,7 @@ var testCasesLoadConfig = map[string]loadConfigTest{
 	},
 	"single_connection_with_default_options_and_workspace_override_terminal_config": { // fixed
 		steampipeDir: "testdata/connection_config/single_connection_with_default_options",
-		workspaceDir: "testdata/workspaces/override_terminal_config",
+		workspaceDir: "testdata/load_config_test/override_terminal_config",
 		expected: &SteampipeConfig{
 			Connections: map[string]*modconfig.Connection{
 				"a": {
@@ -499,7 +499,7 @@ func TestLoadConfig(t *testing.T) {
 		// default workspoace to empty dir
 		workspaceDir := test.workspaceDir
 		if workspaceDir == "" {
-			workspaceDir = "testdata/workspaces/empty"
+			workspaceDir = "testdata/load_config_test/empty"
 		}
 		steampipeDir, err := filepath.Abs(test.steampipeDir)
 		if err != nil {