apply: to check module requirements before run

cmd/apply.go

@@ -29,6 +29,9 @@ var applyCmd = &cobra.Command{
 			log.Println(err)
 			rootDir = projectconfig.RootDir
 		}
-		apply.Apply(rootDir, applyConfigPath, applyEnvironments)
+		applyErr := apply.Apply(rootDir, applyConfigPath, applyEnvironments)
+		if applyErr != nil {
+			log.Fatal(applyErr)
+		}
 	},
 }

internal/apply/apply.go

@@ -1,6 +1,7 @@
 package apply
 
 import (
+	"errors"
 	"fmt"
 	"path/filepath"
 
@@ -13,13 +14,15 @@ import (
 	"github.com/commitdev/zero/internal/util"
 	"github.com/hashicorp/terraform/dag"
 
+	"github.com/commitdev/zero/internal/config/moduleconfig"
 	"github.com/commitdev/zero/internal/config/projectconfig"
 	"github.com/commitdev/zero/pkg/util/exit"
 	"github.com/commitdev/zero/pkg/util/flog"
 	"github.com/manifoldco/promptui"
 )
 
-func Apply(rootDir string, configPath string, environments []string) {
+func Apply(rootDir string, configPath string, environments []string) error {
+	var errs []error
 	if strings.Trim(configPath, " ") == "" {
 		exit.Fatal("config path cannot be empty!")
 	}
@@ -33,6 +36,18 @@ Only a single environment may be suitable for an initial test, but for a real sy
 		environments = promptEnvironments()
 	}
 
+	flog.Infof(":mag: checking project %s's module requirements.", projectConfig.Name)
+
+	errs = modulesWalkCmd("check", rootDir, projectConfig, "check", environments, false)
+	// Check operation walks through all modules and can return multiple errors
+	if len(errs) > 0 {
+		msg := ""
+		for i := 0; i < len(errs); i++ {
+			msg += "\t" + errs[i].Error()
+		}
+		return errors.New(fmt.Sprintf("Module checks failed: \n%s", msg))
+	}
+
 	flog.Infof(":tada: Bootstrapping project %s. Please use the zero-project.yml file to modify the project as needed.", projectConfig.Name)
 
 	flog.Infof("Cloud provider: %s", "AWS") // will this come from the config?
@@ -41,21 +56,27 @@ Only a single environment may be suitable for an initial test, but for a real sy
 
 	flog.Infof("Infrastructure executor: %s", "Terraform")
 
-	applyAll(rootDir, *projectConfig, environments)
+	errs = modulesWalkCmd("apply", rootDir, projectConfig, "apply", environments, true)
+	if len(errs) > 0 {
+		return errors.New(fmt.Sprintf("Module Apply failed: %s", errs[0]))
+	}
 
 	flog.Infof(":check_mark_button: Done.")
 
-	summarizeAll(rootDir, *projectConfig, environments)
+	flog.Infof("Your projects and infrastructure have been successfully created.  Here are some useful links and commands to get you started:")
+	errs = modulesWalkCmd("summary", rootDir, projectConfig, "summary", environments, true)
+	if len(errs) > 0 {
+		return errors.New(fmt.Sprintf("Module summary failed: %s", errs[0]))
+	}
+	return nil
 }
 
-func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEnvironments []string) {
-	environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ","))
-
+func modulesWalkCmd(lifecycleName string, dir string, projectConfig *projectconfig.ZeroProjectConfig, operation string, environments []string, bailOnError bool) []error {
+	var moduleErrors []error
 	graph := projectConfig.GetDAG()
-
-	// Walk the graph of modules and run `make`
 	root := []dag.Vertex{projectconfig.GraphRootName}
-	graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error {
+	environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(environments, ","))
+	err := graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error {
 		// Don't process the root
 		if depth == 0 {
 			return nil
@@ -89,10 +110,58 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn
 		envVarTranslationMap := modConfig.GetParamEnvVarTranslationMap()
 		envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList, envVarTranslationMap)
 		flog.Debugf("Env injected: %#v", envList)
-		flog.Infof("Executing apply command for %s...", modConfig.Name)
-		util.ExecuteCommand(exec.Command("make"), modulePath, envList)
+
+		// only print msg for apply, or else it gets a little spammy
+		if lifecycleName == "apply" {
+			flog.Infof("Executing %s command for %s...", lifecycleName, modConfig.Name)
+		}
+		operationCommand := getModuleOperationCommand(modConfig, operation)
+		execErr := util.ExecuteCommand(exec.Command(operationCommand[0], operationCommand[1:]...), modulePath, envList)
+		if execErr != nil {
+			formatedErr := errors.New(fmt.Sprintf("Module (%s) %s", modConfig.Name, execErr.Error()))
+			if bailOnError {
+				return formatedErr
+			} else {
+				moduleErrors = append(moduleErrors, formatedErr)
+			}
+		}
 		return nil
 	})
+	if err != nil {
+		moduleErrors = append(moduleErrors, err)
+	}
+
+	return moduleErrors
+}
+
+func getModuleOperationCommand(mod moduleconfig.ModuleConfig, operation string) (operationCommand []string) {
+	defaultCheck := []string{"make", "check"}
+	defaultApply := []string{"make"}
+	defaultSummary := []string{"make", "summary"}
+
+	switch operation {
+	case "check":
+		if mod.Commands.Check != "" {
+			operationCommand = strings.Split(mod.Commands.Check, " ")
+		} else {
+			operationCommand = defaultCheck
+		}
+	case "apply":
+		if mod.Commands.Apply != "" {
+			operationCommand = strings.Split(mod.Commands.Apply, " ")
+		} else {
+			operationCommand = defaultApply
+		}
+	case "summary":
+		if mod.Commands.Summary != "" {
+			operationCommand = strings.Split(mod.Commands.Summary, " ")
+		} else {
+			operationCommand = defaultSummary
+		}
+	default:
+		panic("Unexpected operation")
+	}
+	return operationCommand
 }
 
 // promptEnvironments Prompts the user for the environments to apply against and returns a slice of strings representing the environments
@@ -125,47 +194,3 @@ func validateEnvironments(applyEnvironments []string) {
 		}
 	}
 }
-
-func summarizeAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEnvironments []string) {
-	flog.Infof("Your projects and infrastructure have been successfully created.  Here are some useful links and commands to get you started:")
-
-	graph := projectConfig.GetDAG()
-
-	// Walk the graph of modules and run `make summary`
-	root := []dag.Vertex{projectconfig.GraphRootName}
-	graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error {
-		// Don't process the root
-		if depth == 0 {
-			return nil
-		}
-
-		name := v.(string)
-		mod := projectConfig.Modules[name]
-		// Add env vars for the makefile
-		envList := []string{
-			fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ",")),
-			fmt.Sprintf("REPOSITORY=%s", mod.Files.Repository),
-			fmt.Sprintf("PROJECT_NAME=%s", projectConfig.Name),
-		}
-
-		modulePath := module.GetSourceDir(mod.Files.Source)
-		// Passed in `dir` will only be used to find the project path, not the module path,
-		// unless the module path is relative
-		if module.IsLocal(mod.Files.Source) && !filepath.IsAbs(modulePath) {
-			modulePath = filepath.Join(dir, modulePath)
-		}
-		flog.Debugf("Loaded module: %s from %s", name, modulePath)
-
-		modConfig, err := module.ParseModuleConfig(modulePath)
-		if err != nil {
-			exit.Fatal("Failed to load module config, credentials cannot be injected properly")
-		}
-		envVarTranslationMap := modConfig.GetParamEnvVarTranslationMap()
-		envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList, envVarTranslationMap)
-		flog.Debugf("Env injected: %#v", envList)
-		util.ExecuteCommand(exec.Command("make", "summary"), modulePath, envList)
-		return nil
-	})
-
-	flog.Infof("Happy coding! :smile:")
-}

internal/apply/apply_test.go

@@ -13,20 +13,13 @@ import (
 )
 
 func TestApply(t *testing.T) {
-	dir := "../../tests/test_data/apply/"
 	applyConfigPath := constants.ZeroProjectYml
 	applyEnvironments := []string{"staging", "production"}
-
-	tmpDir := filepath.Join(os.TempDir(), "apply")
-
-	err := os.RemoveAll(tmpDir)
-	assert.NoError(t, err)
-
-	err = shutil.CopyTree(dir, tmpDir, nil)
-	assert.NoError(t, err)
+	var tmpDir string
 
 	t.Run("Should run apply and execute make on each folder module", func(t *testing.T) {
-		apply.Apply(tmpDir, applyConfigPath, applyEnvironments)
+		tmpDir = setupTmpDir(t, "../../tests/test_data/apply/")
+		err := apply.Apply(tmpDir, applyConfigPath, applyEnvironments)
 		assert.FileExists(t, filepath.Join(tmpDir, "project1/project.out"))
 		assert.FileExists(t, filepath.Join(tmpDir, "project2/project.out"))
 
@@ -37,6 +30,13 @@ func TestApply(t *testing.T) {
 		content, err = ioutil.ReadFile(filepath.Join(tmpDir, "project2/project.out"))
 		assert.NoError(t, err)
 		assert.Equal(t, "baz: qux\n", string(content))
+
+	})
+
+	t.Run("Moudles runs command overides", func(t *testing.T) {
+		content, err := ioutil.ReadFile(filepath.Join(tmpDir, "project2/check.out"))
+		assert.NoError(t, err)
+		assert.Equal(t, "custom check\n", string(content))
 	})
 
 	t.Run("Zero apply honors the envVarName overwrite from module definition", func(t *testing.T) {
@@ -45,4 +45,26 @@ func TestApply(t *testing.T) {
 		assert.Equal(t, "envVarName of viaEnvVarName: baz\n", string(content))
 	})
 
+	t.Run("Moudles with failing checks should return error", func(t *testing.T) {
+		tmpDir = setupTmpDir(t, "../../tests/test_data/apply-failing/")
+
+		err := apply.Apply(tmpDir, applyConfigPath, applyEnvironments)
+		assert.Regexp(t, "^Module checks failed:", err.Error())
+		assert.Regexp(t, "Module \\(project1\\)", err.Error())
+		assert.Regexp(t, "Module \\(project2\\)", err.Error())
+		assert.Regexp(t, "Module \\(project3\\)", err.Error())
+	})
+
+}
+
+func setupTmpDir(t *testing.T, exampleDirPath string) string {
+	var err error
+	tmpDir := filepath.Join(os.TempDir(), "apply")
+
+	err = os.RemoveAll(tmpDir)
+	assert.NoError(t, err)
+
+	err = shutil.CopyTree(exampleDirPath, tmpDir, nil)
+	assert.NoError(t, err)
+	return tmpDir
 }

internal/config/moduleconfig/module_config.go

@@ -22,14 +22,21 @@ type ModuleConfig struct {
 	Name                string
 	Description         string
 	Author              string
-	DependsOn           []string `yaml:"dependsOn,omitempty"`
+	Commands            ModuleCommands `yaml:"commands,omitempty"`
+	DependsOn           []string       `yaml:"dependsOn,omitempty"`
 	TemplateConfig      `yaml:"template"`
 	RequiredCredentials []string           `yaml:"requiredCredentials"`
 	ZeroVersion         VersionConstraints `yaml:"zeroVersion,omitempty"`
 	Parameters          []Parameter
 	Conditions          []Condition `yaml:"conditions,omitempty"`
 }
 
+type ModuleCommands struct {
+	Apply   string `yaml:"apply,omitempty"`
+	Check   string `yaml:"check,omitempty"`
+	Summary string `yaml:"summary,omitempty"`
+}
+
 func checkVersionAgainstConstrains(vc VersionConstraints, versionString string) bool {
 	v, err := goVerson.NewVersion(versionString)
 	if err != nil {

internal/module/module_test.go

@@ -89,6 +89,11 @@ func TestParseModuleConfig(t *testing.T) {
 		assert.Equal(t, []string{"<%", "%>"}, mod.TemplateConfig.Delimiters)
 	})
 
+	t.Run("Parsing commands", func(t *testing.T) {
+		checkCommand := mod.Commands.Check
+		assert.Equal(t, "ls", checkCommand)
+	})
+
 	t.Run("Parsing zero version constraints", func(t *testing.T) {
 		moduleConstraints := mod.ZeroVersion.Constraints.String()
 		assert.Equal(t, ">= 3.0.0, < 4.0.0", moduleConstraints)

internal/util/util.go

@@ -3,6 +3,8 @@ package util
 // @TODO split up and move into /pkg directory
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -11,8 +13,10 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"strconv"
 	"strings"
+	"syscall"
 	"text/template"
 
 	"github.com/google/uuid"
@@ -48,7 +52,7 @@ func GetCwd() string {
 	return dir
 }
 
-func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) {
+func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) error {
 
 	cmd.Dir = pathPrefix
 	if !filepath.IsAbs(pathPrefix) {
@@ -60,6 +64,7 @@ func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) {
 	stderrPipe, _ := cmd.StderrPipe()
 
 	var errStdout, errStderr error
+	errContent := new(bytes.Buffer)
 
 	cmd.Env = os.Environ()
 	if envars != nil {
@@ -68,19 +73,35 @@ func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) {
 
 	err := cmd.Start()
 	if err != nil {
-		log.Fatalf("Starting command failed: %v\n", err)
+		return err
 	}
 
 	go func() {
 		_, errStdout = io.Copy(os.Stdout, stdoutPipe)
 	}()
 	go func() {
-		_, errStderr = io.Copy(os.Stderr, stderrPipe)
+		stdErr := io.MultiWriter(errContent, os.Stderr)
+		_, errStderr = io.Copy(stdErr, stderrPipe)
 	}()
 
 	err = cmd.Wait()
 	if err != nil {
-		log.Fatalf("Executing command failed: %v\n", err)
+		// Detecting and returning the makefile error to cmd
+		// Passing alone makefile stderr as error message, otherwise it just says "exit status 2"
+		if exitError, ok := err.(*exec.ExitError); ok {
+			ws := exitError.Sys().(syscall.WaitStatus)
+			exitCode := ws.ExitStatus()
+			if exitCode == 2 {
+				stderrOut := errContent.String()
+				isMissingTarget, _ := regexp.MatchString("No rule to make target", stderrOut)
+				if isMissingTarget {
+					return errors.New("Module missing mandatory targets, this is likely an issue with the module itself.")
+				}
+				return errors.New(stderrOut)
+			}
+		}
+
+		return errors.New(errContent.String())
 	}
 
 	if errStdout != nil {
@@ -90,6 +111,7 @@ func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) {
 	if errStderr != nil {
 		log.Printf("Failed to capture stderr: %v\n", errStderr)
 	}
+	return nil
 }
 
 // ExecuteCommandOutput runs the command and returns its

tests/test_data/apply-failing/project1/Makefile

@@ -0,0 +1,8 @@
+current_dir:
+	@echo "foo: ${foo}" > project.out
+	@echo "repo: ${REPOSITORY}" >> project.out
+
+summary:
+
+check:
+	@$(error "Failure 1 of 2")

tests/test_data/apply-failing/project1/project.out

@@ -0,0 +1,2 @@
+foo: bar
+repo: github.com/commitdev/project1