diff --git a/cmd/apply.go b/cmd/apply.go index 32d86ec48..705eff747 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -29,6 +29,9 @@ var applyCmd = &cobra.Command{ log.Println(err) rootDir = projectconfig.RootDir } - apply.Apply(rootDir, applyConfigPath, applyEnvironments) + applyErr := apply.Apply(rootDir, applyConfigPath, applyEnvironments) + if applyErr != nil { + log.Fatal(applyErr) + } }, } diff --git a/docs/module-definition.md b/docs/module-definition.md index f75378e82..dec427286 100644 --- a/docs/module-definition.md +++ b/docs/module-definition.md @@ -10,9 +10,17 @@ It also declares the module's dependencies to determine the order of execution | `author` | string | Author of the module | | `icon` | string | Path to logo image | | `parameters` | list(Parameter) | Parameters to prompt users | +| `commands` | Commands | Commands to use instead of makefile defaults | | `zeroVersion` | string([go-semver])| Zero versions its compatible with | +### Commands +Commands are the lifecycle of `zero apply`, it will run all module's `check phase`, then once satisfied run in sequence `apply phase` then if successful run `summary phase`. +| Parameters | Type | Default | Description | +|------------|--------|----------------|--------------------------------------------------------------------------| +| `check` | string | `make check` | Command to check module requirements. check is satisfied if exit code is 0 eg: `sh check-token.sh`, `zero apply` will check all modules before executing | +| `apply` | string | `make` | Command to execute the project provisioning. | +| `summary` | string | `make summary` | Command to summarize to users the module's output and next steps. | ### Template | Parameters | Type | Description | |--------------|---------|-----------------------------------------------------------------------| diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 782153369..c60303197 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -1,6 +1,7 @@ package apply import ( + "errors" "fmt" "path/filepath" @@ -13,13 +14,15 @@ import ( "github.com/commitdev/zero/internal/util" "github.com/hashicorp/terraform/dag" + "github.com/commitdev/zero/internal/config/moduleconfig" "github.com/commitdev/zero/internal/config/projectconfig" "github.com/commitdev/zero/pkg/util/exit" "github.com/commitdev/zero/pkg/util/flog" "github.com/manifoldco/promptui" ) -func Apply(rootDir string, configPath string, environments []string) { +func Apply(rootDir string, configPath string, environments []string) error { + var errs []error if strings.Trim(configPath, " ") == "" { exit.Fatal("config path cannot be empty!") } @@ -33,6 +36,18 @@ Only a single environment may be suitable for an initial test, but for a real sy environments = promptEnvironments() } + flog.Infof(":mag: checking project %s's module requirements.", projectConfig.Name) + + errs = modulesWalkCmd("check", rootDir, projectConfig, "check", environments, false, false) + // Check operation walks through all modules and can return multiple errors + if len(errs) > 0 { + msg := "" + for i := 0; i < len(errs); i++ { + msg += "- " + errs[i].Error() + } + return errors.New(fmt.Sprintf("The following Module check(s) failed: \n%s", msg)) + } + flog.Infof(":tada: Bootstrapping project %s. Please use the zero-project.yml file to modify the project as needed.", projectConfig.Name) flog.Infof("Cloud provider: %s", "AWS") // will this come from the config? @@ -41,21 +56,27 @@ Only a single environment may be suitable for an initial test, but for a real sy flog.Infof("Infrastructure executor: %s", "Terraform") - applyAll(rootDir, *projectConfig, environments) + errs = modulesWalkCmd("apply", rootDir, projectConfig, "apply", environments, true, true) + if len(errs) > 0 { + return errors.New(fmt.Sprintf("Module Apply failed: %s", errs[0])) + } flog.Infof(":check_mark_button: Done.") - summarizeAll(rootDir, *projectConfig, environments) + flog.Infof("Your projects and infrastructure have been successfully created. Here are some useful links and commands to get you started:") + errs = modulesWalkCmd("summary", rootDir, projectConfig, "summary", environments, true, true) + if len(errs) > 0 { + return errors.New(fmt.Sprintf("Module summary failed: %s", errs[0])) + } + return nil } -func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEnvironments []string) { - environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ",")) - +func modulesWalkCmd(lifecycleName string, dir string, projectConfig *projectconfig.ZeroProjectConfig, operation string, environments []string, bailOnError bool, shouldPipeStderr bool) []error { + var moduleErrors []error graph := projectConfig.GetDAG() - - // Walk the graph of modules and run `make` root := []dag.Vertex{projectconfig.GraphRootName} - graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error { + environmentArg := fmt.Sprintf("ENVIRONMENT=%s", strings.Join(environments, ",")) + err := graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error { // Don't process the root if depth == 0 { return nil @@ -83,16 +104,64 @@ func applyAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEn // and we should redownload the module for the user modConfig, err := module.ParseModuleConfig(modulePath) if err != nil { - exit.Fatal("Failed to load module config, credentials cannot be injected properly") + exit.Fatal("Failed to load Module: %s", err) } envVarTranslationMap := modConfig.GetParamEnvVarTranslationMap() envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList, envVarTranslationMap) flog.Debugf("Env injected: %#v", envList) - flog.Infof("Executing apply command for %s...", modConfig.Name) - util.ExecuteCommand(exec.Command("make"), modulePath, envList) + + // only print msg for apply, or else it gets a little spammy + if lifecycleName == "apply" { + flog.Infof("Executing %s command for %s...", lifecycleName, modConfig.Name) + } + operationCommand := getModuleOperationCommand(modConfig, operation) + execErr := util.ExecuteCommand(exec.Command(operationCommand[0], operationCommand[1:]...), modulePath, envList, shouldPipeStderr) + if execErr != nil { + formatedErr := errors.New(fmt.Sprintf("Module (%s) %s", modConfig.Name, execErr.Error())) + if bailOnError { + return formatedErr + } else { + moduleErrors = append(moduleErrors, formatedErr) + } + } return nil }) + if err != nil { + moduleErrors = append(moduleErrors, err) + } + + return moduleErrors +} + +func getModuleOperationCommand(mod moduleconfig.ModuleConfig, operation string) (operationCommand []string) { + defaultCheck := []string{"make", "check"} + defaultApply := []string{"make"} + defaultSummary := []string{"make", "summary"} + + switch operation { + case "check": + if mod.Commands.Check != "" { + operationCommand = []string{"sh", "-c", mod.Commands.Check} + } else { + operationCommand = defaultCheck + } + case "apply": + if mod.Commands.Apply != "" { + operationCommand = []string{"sh", "-c", mod.Commands.Apply} + } else { + operationCommand = defaultApply + } + case "summary": + if mod.Commands.Summary != "" { + operationCommand = []string{"sh", "-c", mod.Commands.Summary} + } else { + operationCommand = defaultSummary + } + default: + panic("Unexpected operation") + } + return operationCommand } // promptEnvironments Prompts the user for the environments to apply against and returns a slice of strings representing the environments @@ -125,47 +194,3 @@ func validateEnvironments(applyEnvironments []string) { } } } - -func summarizeAll(dir string, projectConfig projectconfig.ZeroProjectConfig, applyEnvironments []string) { - flog.Infof("Your projects and infrastructure have been successfully created. Here are some useful links and commands to get you started:") - - graph := projectConfig.GetDAG() - - // Walk the graph of modules and run `make summary` - root := []dag.Vertex{projectconfig.GraphRootName} - graph.DepthFirstWalk(root, func(v dag.Vertex, depth int) error { - // Don't process the root - if depth == 0 { - return nil - } - - name := v.(string) - mod := projectConfig.Modules[name] - // Add env vars for the makefile - envList := []string{ - fmt.Sprintf("ENVIRONMENT=%s", strings.Join(applyEnvironments, ",")), - fmt.Sprintf("REPOSITORY=%s", mod.Files.Repository), - fmt.Sprintf("PROJECT_NAME=%s", projectConfig.Name), - } - - modulePath := module.GetSourceDir(mod.Files.Source) - // Passed in `dir` will only be used to find the project path, not the module path, - // unless the module path is relative - if module.IsLocal(mod.Files.Source) && !filepath.IsAbs(modulePath) { - modulePath = filepath.Join(dir, modulePath) - } - flog.Debugf("Loaded module: %s from %s", name, modulePath) - - modConfig, err := module.ParseModuleConfig(modulePath) - if err != nil { - exit.Fatal("Failed to load module config, credentials cannot be injected properly") - } - envVarTranslationMap := modConfig.GetParamEnvVarTranslationMap() - envList = util.AppendProjectEnvToCmdEnv(mod.Parameters, envList, envVarTranslationMap) - flog.Debugf("Env injected: %#v", envList) - util.ExecuteCommand(exec.Command("make", "summary"), modulePath, envList) - return nil - }) - - flog.Infof("Happy coding! :smile:") -} diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index a4c1b41e4..e5ecc0359 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -13,20 +13,13 @@ import ( ) func TestApply(t *testing.T) { - dir := "../../tests/test_data/apply/" applyConfigPath := constants.ZeroProjectYml applyEnvironments := []string{"staging", "production"} - - tmpDir := filepath.Join(os.TempDir(), "apply") - - err := os.RemoveAll(tmpDir) - assert.NoError(t, err) - - err = shutil.CopyTree(dir, tmpDir, nil) - assert.NoError(t, err) + var tmpDir string t.Run("Should run apply and execute make on each folder module", func(t *testing.T) { - apply.Apply(tmpDir, applyConfigPath, applyEnvironments) + tmpDir = setupTmpDir(t, "../../tests/test_data/apply/") + err := apply.Apply(tmpDir, applyConfigPath, applyEnvironments) assert.FileExists(t, filepath.Join(tmpDir, "project1/project.out")) assert.FileExists(t, filepath.Join(tmpDir, "project2/project.out")) @@ -37,6 +30,13 @@ func TestApply(t *testing.T) { content, err = ioutil.ReadFile(filepath.Join(tmpDir, "project2/project.out")) assert.NoError(t, err) assert.Equal(t, "baz: qux\n", string(content)) + + }) + + t.Run("Modules runs command overides", func(t *testing.T) { + content, err := ioutil.ReadFile(filepath.Join(tmpDir, "project2/check.out")) + assert.NoError(t, err) + assert.Equal(t, "custom check\n", string(content)) }) t.Run("Zero apply honors the envVarName overwrite from module definition", func(t *testing.T) { @@ -45,4 +45,26 @@ func TestApply(t *testing.T) { assert.Equal(t, "envVarName of viaEnvVarName: baz\n", string(content)) }) + t.Run("Modules with failing checks should return error", func(t *testing.T) { + tmpDir = setupTmpDir(t, "../../tests/test_data/apply-failing/") + + err := apply.Apply(tmpDir, applyConfigPath, applyEnvironments) + assert.Regexp(t, "^The following Module check\\(s\\) failed:", err.Error()) + assert.Regexp(t, "Module \\(project1\\)", err.Error()) + assert.Regexp(t, "Module \\(project2\\)", err.Error()) + assert.Regexp(t, "Module \\(project3\\)", err.Error()) + }) + +} + +func setupTmpDir(t *testing.T, exampleDirPath string) string { + var err error + tmpDir := filepath.Join(os.TempDir(), "apply") + + err = os.RemoveAll(tmpDir) + assert.NoError(t, err) + + err = shutil.CopyTree(exampleDirPath, tmpDir, nil) + assert.NoError(t, err) + return tmpDir } diff --git a/internal/config/moduleconfig/module_config.go b/internal/config/moduleconfig/module_config.go index df1199ca5..f040a0595 100644 --- a/internal/config/moduleconfig/module_config.go +++ b/internal/config/moduleconfig/module_config.go @@ -22,7 +22,8 @@ type ModuleConfig struct { Name string Description string Author string - DependsOn []string `yaml:"dependsOn,omitempty"` + Commands ModuleCommands `yaml:"commands,omitempty"` + DependsOn []string `yaml:"dependsOn,omitempty"` TemplateConfig `yaml:"template"` RequiredCredentials []string `yaml:"requiredCredentials"` ZeroVersion VersionConstraints `yaml:"zeroVersion,omitempty"` @@ -30,6 +31,12 @@ type ModuleConfig struct { Conditions []Condition `yaml:"conditions,omitempty"` } +type ModuleCommands struct { + Apply string `yaml:"apply,omitempty"` + Check string `yaml:"check,omitempty"` + Summary string `yaml:"summary,omitempty"` +} + func checkVersionAgainstConstrains(vc VersionConstraints, versionString string) bool { v, err := goVerson.NewVersion(versionString) if err != nil { diff --git a/internal/module/module_test.go b/internal/module/module_test.go index 21d0df699..34ae1111b 100644 --- a/internal/module/module_test.go +++ b/internal/module/module_test.go @@ -89,6 +89,11 @@ func TestParseModuleConfig(t *testing.T) { assert.Equal(t, []string{"<%", "%>"}, mod.TemplateConfig.Delimiters) }) + t.Run("Parsing commands", func(t *testing.T) { + checkCommand := mod.Commands.Check + assert.Equal(t, "ls", checkCommand) + }) + t.Run("Parsing zero version constraints", func(t *testing.T) { moduleConstraints := mod.ZeroVersion.Constraints.String() assert.Equal(t, ">= 3.0.0, < 4.0.0", moduleConstraints) diff --git a/internal/util/util.go b/internal/util/util.go index 8dc5826c4..5dc13c1e7 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -3,6 +3,8 @@ package util // @TODO split up and move into /pkg directory import ( + "bytes" + "errors" "fmt" "io" "log" @@ -11,8 +13,10 @@ import ( "path" "path/filepath" "reflect" + "regexp" "strconv" "strings" + "syscall" "text/template" "github.com/google/uuid" @@ -48,7 +52,7 @@ func GetCwd() string { return dir } -func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) { +func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string, shouldPipeStdErr bool) error { cmd.Dir = pathPrefix if !filepath.IsAbs(pathPrefix) { @@ -60,6 +64,7 @@ func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) { stderrPipe, _ := cmd.StderrPipe() var errStdout, errStderr error + errContent := new(bytes.Buffer) cmd.Env = os.Environ() if envars != nil { @@ -68,19 +73,39 @@ func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) { err := cmd.Start() if err != nil { - log.Fatalf("Starting command failed: %v\n", err) + return err } go func() { _, errStdout = io.Copy(os.Stdout, stdoutPipe) }() go func() { - _, errStderr = io.Copy(os.Stderr, stderrPipe) + stderrStreams := []io.Writer{errContent} + if shouldPipeStdErr { + stderrStreams = append(stderrStreams, os.Stderr) + } + stdErr := io.MultiWriter(stderrStreams...) + _, errStderr = io.Copy(stdErr, stderrPipe) }() err = cmd.Wait() if err != nil { - log.Fatalf("Executing command failed: %v\n", err) + // Detecting and returning the makefile error to cmd + // Passing alone makefile stderr as error message, otherwise it just says "exit status 2" + if exitError, ok := err.(*exec.ExitError); ok { + ws := exitError.Sys().(syscall.WaitStatus) + exitCode := ws.ExitStatus() + if exitCode == 2 { + stderrOut := errContent.String() + isMissingTarget, _ := regexp.MatchString("No rule to make target", stderrOut) + if isMissingTarget { + return errors.New("Module missing mandatory targets, this is likely an issue with the module itself.") + } + return errors.New(stderrOut) + } + } + + return errors.New(errContent.String()) } if errStdout != nil { @@ -90,6 +115,7 @@ func ExecuteCommand(cmd *exec.Cmd, pathPrefix string, envars []string) { if errStderr != nil { log.Printf("Failed to capture stderr: %v\n", errStderr) } + return nil } // ExecuteCommandOutput runs the command and returns its diff --git a/tests/test_data/apply-failing/project1/Makefile b/tests/test_data/apply-failing/project1/Makefile new file mode 100644 index 000000000..b3236e75a --- /dev/null +++ b/tests/test_data/apply-failing/project1/Makefile @@ -0,0 +1,8 @@ +current_dir: + @echo "foo: ${foo}" > project.out + @echo "repo: ${REPOSITORY}" >> project.out + +summary: + +check: + @$(error "Failure 1 of 2") diff --git a/tests/test_data/apply-failing/project1/project.out b/tests/test_data/apply-failing/project1/project.out new file mode 100644 index 000000000..028efdf7e --- /dev/null +++ b/tests/test_data/apply-failing/project1/project.out @@ -0,0 +1,2 @@ +foo: bar +repo: github.com/commitdev/project1 diff --git a/tests/test_data/apply-failing/project1/zero-module.yml b/tests/test_data/apply-failing/project1/zero-module.yml new file mode 100644 index 000000000..5d6914f9b --- /dev/null +++ b/tests/test_data/apply-failing/project1/zero-module.yml @@ -0,0 +1,19 @@ +name: project1 +description: 'project1' +author: 'Commit' + +template: + strictMode: true + delimiters: + - "<%" + - "%>" + inputDir: '.' + outputDir: 'test' + +requiredCredentials: + - aws + - github + +parameters: + - field: foo + label: foo diff --git a/tests/test_data/apply-failing/project2/Makefile b/tests/test_data/apply-failing/project2/Makefile new file mode 100644 index 000000000..808931db0 --- /dev/null +++ b/tests/test_data/apply-failing/project2/Makefile @@ -0,0 +1,10 @@ +REQUIRED_BINS := ls nonexisting-binary + +current_dir: + @echo "baz: ${baz}" > project.out + +summary: + +check: + $(foreach bin, $(REQUIRED_BINS),\ + $(if $(shell command -v $(bin) 2> /dev/null),$(info Found `$(bin)`),$(error Please install `$(bin)`))) diff --git a/tests/test_data/apply-failing/project2/project.out b/tests/test_data/apply-failing/project2/project.out new file mode 100644 index 000000000..48416721f --- /dev/null +++ b/tests/test_data/apply-failing/project2/project.out @@ -0,0 +1 @@ +baz: qux diff --git a/tests/test_data/apply-failing/project2/zero-module.yml b/tests/test_data/apply-failing/project2/zero-module.yml new file mode 100644 index 000000000..c50d5df76 --- /dev/null +++ b/tests/test_data/apply-failing/project2/zero-module.yml @@ -0,0 +1,19 @@ +name: project2 +description: 'project2' +author: 'Commit' + +template: + strictMode: true + delimiters: + - "<%" + - "%>" + inputDir: '.' + outputDir: 'test' + +requiredCredentials: + - aws + - github + +parameters: + - field: baz + label: baz diff --git a/tests/test_data/apply-failing/project3/Makefile b/tests/test_data/apply-failing/project3/Makefile new file mode 100644 index 000000000..9a725c790 --- /dev/null +++ b/tests/test_data/apply-failing/project3/Makefile @@ -0,0 +1,5 @@ +REQUIRED_BINS := ls nonexisting-binary + +current_dir: + +summary: diff --git a/tests/test_data/apply-failing/project3/check.sh b/tests/test_data/apply-failing/project3/check.sh new file mode 100644 index 000000000..376dcab21 --- /dev/null +++ b/tests/test_data/apply-failing/project3/check.sh @@ -0,0 +1 @@ +>&2 echo "Check script erroring out";exit 1; \ No newline at end of file diff --git a/tests/test_data/apply-failing/project3/project.out b/tests/test_data/apply-failing/project3/project.out new file mode 100644 index 000000000..48416721f --- /dev/null +++ b/tests/test_data/apply-failing/project3/project.out @@ -0,0 +1 @@ +baz: qux diff --git a/tests/test_data/apply-failing/project3/zero-module.yml b/tests/test_data/apply-failing/project3/zero-module.yml new file mode 100644 index 000000000..826ffc129 --- /dev/null +++ b/tests/test_data/apply-failing/project3/zero-module.yml @@ -0,0 +1,21 @@ +name: project3 +description: 'project3' +author: 'Commit' + +commands: + check: sh check.sh +template: + strictMode: true + delimiters: + - "<%" + - "%>" + inputDir: '.' + outputDir: 'test' + +requiredCredentials: + - aws + - github + +parameters: + - field: baz + label: baz diff --git a/tests/test_data/apply-failing/zero-project.yml b/tests/test_data/apply-failing/zero-project.yml new file mode 100644 index 000000000..dff2c5b60 --- /dev/null +++ b/tests/test_data/apply-failing/zero-project.yml @@ -0,0 +1,22 @@ +name: sample_project + +modules: + project1: + parameters: + foo: bar + files: + dir: project1 + repo: github.com/commitdev/project1 + source: project1 + project2: + parameters: + baz: qux + files: + dir: project2 + repo: github.com/commitdev/project2 + source: project2 + project3: + files: + dir: project3 + repo: github.com/commitdev/project3 + source: project3 diff --git a/tests/test_data/apply/project1/Makefile b/tests/test_data/apply/project1/Makefile index 78158f243..d7b3251d8 100644 --- a/tests/test_data/apply/project1/Makefile +++ b/tests/test_data/apply/project1/Makefile @@ -4,3 +4,5 @@ current_dir: @echo "envVarName of viaEnvVarName: ${viaEnvVarName}" >> feature.out summary: + +check: diff --git a/tests/test_data/apply/project2/Makefile b/tests/test_data/apply/project2/Makefile index 41f42f07b..d824e2d06 100644 --- a/tests/test_data/apply/project2/Makefile +++ b/tests/test_data/apply/project2/Makefile @@ -2,3 +2,5 @@ current_dir: @echo "baz: ${baz}" > project.out summary: + +check: diff --git a/tests/test_data/apply/project2/check.sh b/tests/test_data/apply/project2/check.sh new file mode 100644 index 000000000..b73208798 --- /dev/null +++ b/tests/test_data/apply/project2/check.sh @@ -0,0 +1,2 @@ +pwd +echo "custom check" > check.out \ No newline at end of file diff --git a/tests/test_data/apply/project2/zero-module.yml b/tests/test_data/apply/project2/zero-module.yml index c50d5df76..6ce2ddf03 100644 --- a/tests/test_data/apply/project2/zero-module.yml +++ b/tests/test_data/apply/project2/zero-module.yml @@ -1,7 +1,8 @@ name: project2 description: 'project2' author: 'Commit' - +commands: + check: sh check.sh template: strictMode: true delimiters: diff --git a/tests/test_data/modules/ci/zero-module.yml b/tests/test_data/modules/ci/zero-module.yml index 7e5081368..4e78e7503 100644 --- a/tests/test_data/modules/ci/zero-module.yml +++ b/tests/test_data/modules/ci/zero-module.yml @@ -4,6 +4,8 @@ author: "CI author" icon: "" thumbnail: "" zeroVersion: ">= 3.0.0, < 4.0.0" +commands: + check: ls requiredCredentials: - aws