diff --git a/config.go b/config.go index a71b7f7..4293be5 100644 --- a/config.go +++ b/config.go @@ -14,6 +14,10 @@ import ( "gopkg.in/yaml.v2" ) +var ( + reModuledir = regexp.MustCompile(`^\s*(?:moduledir)\s+['\"]?([^'\"]+)['\"]?`) +) + // readConfigfile creates the ConfigSettings struct from the g10k config file func readConfigfile(configFile string) ConfigSettings { Debugf("Trying to read g10k config file: " + configFile) @@ -51,11 +55,11 @@ func readConfigfile(configFile string) ConfigSettings { config.ModulesCacheDir = checkDirAndCreate(filepath.Join(config.CacheDir, "modules"), "cachedir/modules") config.EnvCacheDir = checkDirAndCreate(filepath.Join(config.CacheDir, "environments"), "cachedir/environments") - if len(config.Forge.Baseurl) == 0 { - config.Forge.Baseurl = "https://forgeapi.puppet.com" + if len(config.ForgeBaseURL) == 0 { + config.ForgeBaseURL = "https://forgeapi.puppet.com" } - //fmt.Println("Forge Baseurl: ", config.Forge.Baseurl) + // fmt.Println("Forge Baseurl: ", config.ForgeBaseURL) // set default timeout to 5 seconds if no timeout setting found if config.Timeout == 0 { @@ -98,6 +102,14 @@ func readConfigfile(configFile string) ConfigSettings { config.MaxExtractworker = 20 } + if len(config.ForgeCacheTTLString) != 0 { + ttl, err := time.ParseDuration(config.ForgeCacheTTLString) + if err != nil { + Fatalf("Error: Can not convert value " + config.ForgeCacheTTLString + " of config setting forge_cache_ttl to a golang Duration. Valid time units are 300ms, 1.5h or 2h45m. In " + configFile) + } + config.ForgeCacheTTL = ttl + } + // check for non-empty config.Deploy which takes precedence over the non-deploy scoped settings // See https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy emptyDeploy := DeploySettings{} @@ -132,7 +144,7 @@ func readConfigfile(configFile string) ConfigSettings { Validatef() } - //fmt.Printf("%+v\n", config) + // fmt.Printf("%+v\n", config) return config } @@ -190,7 +202,6 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc } reEmptyLine := regexp.MustCompile(`^\s*$`) - reModuledir := regexp.MustCompile(`^\s*(?:moduledir)\s+['\"]?([^'\"]+)['\"]?`) reForgeCacheTTL := regexp.MustCompile(`^\s*(?:forge.cache(?:TTL|Ttl))\s+['\"]?([^'\"]+)['\"]?`) reForgeBaseURL := regexp.MustCompile(`^\s*(?:forge.base(?:URL|Url))\s+['\"]?([^'\"]+)['\"]?`) reForgeModule := regexp.MustCompile(`^\s*(?:mod)\s+['\"]?([^'\"]+[-/][^'\"]+)['\"](?:\s*)[,]?(.*)`) @@ -302,6 +313,10 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc if _, ok := puppetFile.gitModules[comp[1]]; ok { Fatalf("Error: Forge Puppet module with same name found in " + pf + " for module " + comp[1] + " line: " + line) } + // the base url in the Puppetfile takes precedence over an base url specified in the g10k config yaml + if len(puppetFile.forgeBaseURL) == 0 { + puppetFile.forgeBaseURL = config.ForgeBaseURL + } puppetFile.forgeModules[comp[1]] = ForgeModule{version: forgeModuleVersion, name: comp[1], author: comp[0], sha256sum: forgeChecksum, moduleDir: moduleDir, sourceBranch: source + "_" + branch} } else if m := reGitModule.FindStringSubmatch(line); len(m) > 1 { gitModuleName := m[1] @@ -431,6 +446,6 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc puppetFile.moduleDirs = moduleDirs puppetFile.sourceBranch = branch - //fmt.Printf("%+v\n", puppetFile) + // fmt.Printf("%+v\n", puppetFile) return puppetFile } diff --git a/forge.go b/forge.go index 2994cc9..e39c445 100644 --- a/forge.go +++ b/forge.go @@ -165,7 +165,7 @@ func doModuleInstallOrNothing(fm ForgeModule) { } func queryForgeAPI(fm ForgeModule) ForgeResult { - baseURL := config.Forge.Baseurl + baseURL := config.ForgeBaseURL if len(fm.baseURL) > 0 { baseURL = fm.baseURL } @@ -276,7 +276,7 @@ func parseForgeAPIResult(json string, fm ForgeModule) ForgeResult { // getMetadataForgeModule queries the configured Puppet Forge and return func getMetadataForgeModule(fm ForgeModule) ForgeModule { - baseURL := config.Forge.Baseurl + baseURL := config.ForgeBaseURL if len(fm.baseURL) > 0 { baseURL = fm.baseURL } @@ -361,7 +361,7 @@ func downloadForgeModule(name string, version string, fm ForgeModule, retryCount fileName := name + "-" + version + ".tar.gz" if !isDir(filepath.Join(config.ForgeCacheDir, name+"-"+version)) { - baseURL := config.Forge.Baseurl + baseURL := config.ForgeBaseURL if len(fm.baseURL) > 0 { baseURL = fm.baseURL } @@ -685,9 +685,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo check4ForgeUpdate(m.name, me.version, latestForgeModules.m[moduleName]) latestForgeModules.RUnlock() } - mutex.Lock() - unchangedModuleDirs = append(unchangedModuleDirs, targetDir) - mutex.Unlock() return } // safe to do, because we ensured in doModuleInstallOrNothing() that -latest exists @@ -714,9 +711,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo } if me.version == m.version { Debugf("Nothing to do, existing Forge module: " + targetDir + " has the same version " + me.version + " as the to be synced version: " + m.version) - mutex.Lock() - unchangedModuleDirs = append(unchangedModuleDirs, targetDir) - mutex.Unlock() return } Infof("Need to sync, because existing Forge module: " + targetDir + " has version " + me.version + " and the to be synced version is: " + m.version) @@ -747,9 +741,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo Infof("Need to sync " + targetDir) if !dryRun { targetDir = checkDirAndCreate(targetDir, "as targetDir for module "+name) - mutex.Lock() - desiredContent = append(desiredContent, targetDir) - mutex.Unlock() var targetDirDevice, workDirDevice uint64 if fileInfo, err := os.Stat(targetDir); err == nil { if fileInfo.Sys() != nil { @@ -785,10 +776,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo if err != nil { Fatalf(funcName + "(): Can't make " + path + " relative to " + resolvedWorkDir + " Error: " + err.Error()) } - mutex.Lock() - desiredContent = append(desiredContent, filepath.Join(targetDir, target)) - //Debugf("adding path to managed content: " + filepath.Join(targetDir, target)) - mutex.Unlock() if info.IsDir() { if target != "." { // skip the root dir diff --git a/g10k.go b/g10k.go index 262bd55..daed7d2 100755 --- a/g10k.go +++ b/g10k.go @@ -58,8 +58,6 @@ var ( maxworker int maxExtractworker int forgeModuleDeprecationNotice string - desiredContent []string - unchangedModuleDirs []string ) // LatestForgeModules contains a map of unique Forge modules @@ -76,7 +74,6 @@ type ConfigSettings struct { ModulesCacheDir string EnvCacheDir string Git Git - Forge Forge Sources map[string]Source Timeout int `yaml:"timeout"` IgnoreUnreachableModules bool `yaml:"ignore_unreachable_modules"` @@ -93,8 +90,11 @@ type ConfigSettings struct { WriteLock string `yaml:"write_lock"` GenerateTypes bool `yaml:"generate_types"` PuppetPath string `yaml:"puppet_path"` - PurgeSkiplist []string `yaml:"purge_skiplist"` + PurgeSkiplist []string `yaml:"purge_skiplist"` CloneGitModules bool `yaml:"clone_git_modules"` + ForgeBaseURL string `yaml:"forge_base_url"` + ForgeCacheTTLString string `yaml:"forge_cache_ttl"` + ForgeCacheTTL time.Duration } // DeploySettings is a struct for settings for controlling how g10k deploys behave. @@ -106,7 +106,7 @@ type DeploySettings struct { WriteLock string `yaml:"write_lock"` GenerateTypes bool `yaml:"generate_types"` PuppetPath string `yaml:"puppet_path"` - PurgeSkiplist []string `yaml:"purge_skiplist"` + PurgeSkiplist []string `yaml:"purge_skiplist"` } // Forge is a simple struct that contains the base URL of @@ -308,10 +308,9 @@ func main() { cachedir = checkDirAndCreate(cachedir, "cachedir default value") } // default purge_levels - forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppet.com"} modulesCacheDir := filepath.Join(cachedir, "modules") envsCacheDir := filepath.Join(cachedir, "environments") - config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: modulesCacheDir, EnvCacheDir: envsCacheDir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported} + config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: modulesCacheDir, EnvCacheDir: envsCacheDir, Sources: sm, ForgeBaseURL: "https://forgeapi.puppet.com", Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported} config.PurgeLevels = []string{"puppetfile"} target = pfLocation puppetfile := readPuppetfile(target, "", "cmdlineparam", "cmdlineparam", false, false) diff --git a/g10k_test.go b/g10k_test.go index 74d5fe2..8d62468 100755 --- a/g10k_test.go +++ b/g10k_test.go @@ -61,9 +61,11 @@ func TestConfigPrefix(t *testing.T) { expected := ConfigSettings{ CacheDir: "/tmp/g10k", ForgeCacheDir: "/tmp/g10k/forge", ModulesCacheDir: "/tmp/g10k/modules", EnvCacheDir: "/tmp/g10k/environments", - Git: Git{privateKey: ""}, - Forge: Forge{Baseurl: "https://forgeapi.puppet.com"}, - Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, + Git: Git{privateKey: ""}, + ForgeCacheTTLString: "24h", + ForgeCacheTTL: 24 * time.Hour, + ForgeBaseURL: "https://forgeapi.puppet.com", + Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, PurgeLevels: []string{"deployment", "puppetfile"}} if !reflect.DeepEqual(got, expected) { @@ -87,9 +89,9 @@ func TestConfigForceForgeVersions(t *testing.T) { expected := ConfigSettings{ CacheDir: "/tmp/g10k", ForgeCacheDir: "/tmp/g10k/forge", ModulesCacheDir: "/tmp/g10k/modules", EnvCacheDir: "/tmp/g10k/environments", - Git: Git{privateKey: ""}, - Forge: Forge{Baseurl: "https://forgeapi.puppet.com"}, - Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, + Git: Git{privateKey: ""}, + ForgeBaseURL: "https://forgeapi.puppet.com", + Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, PurgeLevels: []string{"deployment", "puppetfile"}} if !reflect.DeepEqual(got, expected) { @@ -113,9 +115,9 @@ func TestConfigAddWarning(t *testing.T) { expected := ConfigSettings{ CacheDir: "/tmp/g10k", ForgeCacheDir: "/tmp/g10k/forge", ModulesCacheDir: "/tmp/g10k/modules", EnvCacheDir: "/tmp/g10k/environments", - Git: Git{privateKey: ""}, - Forge: Forge{Baseurl: "https://forgeapi.puppet.com"}, - Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, + Git: Git{privateKey: ""}, + ForgeBaseURL: "https://forgeapi.puppet.com", + Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, PurgeLevels: []string{"deployment", "puppetfile"}} if !reflect.DeepEqual(got, expected) { @@ -140,9 +142,9 @@ func TestConfigSimplePostrunCommand(t *testing.T) { expected := ConfigSettings{ CacheDir: "/tmp/g10k", ForgeCacheDir: "/tmp/g10k/forge", ModulesCacheDir: "/tmp/g10k/modules", EnvCacheDir: "/tmp/g10k/environments", - Git: Git{privateKey: ""}, - Forge: Forge{Baseurl: "https://forgeapi.puppet.com"}, - Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, + Git: Git{privateKey: ""}, + ForgeBaseURL: "https://forgeapi.puppet.com", + Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, PurgeLevels: []string{"deployment", "puppetfile"}, PostRunCommand: postrunCommand} if !reflect.DeepEqual(got, expected) { @@ -167,9 +169,9 @@ func TestConfigPostrunCommand(t *testing.T) { expected := ConfigSettings{ CacheDir: "/tmp/g10k", ForgeCacheDir: "/tmp/g10k/forge", ModulesCacheDir: "/tmp/g10k/modules", EnvCacheDir: "/tmp/g10k/environments", - Git: Git{privateKey: ""}, - Forge: Forge{Baseurl: "https://forgeapi.puppet.com"}, - Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, + Git: Git{privateKey: ""}, + ForgeBaseURL: "https://forgeapi.puppet.com", + Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, PurgeLevels: []string{"deployment", "puppetfile"}, PostRunCommand: postrunCommand} if !reflect.DeepEqual(got, expected) { @@ -193,9 +195,9 @@ func TestConfigDeploy(t *testing.T) { expected := ConfigSettings{ CacheDir: "/tmp/g10k", ForgeCacheDir: "/tmp/g10k/forge", ModulesCacheDir: "/tmp/g10k/modules", EnvCacheDir: "/tmp/g10k/environments", - Git: Git{privateKey: ""}, - Forge: Forge{Baseurl: "https://forgeapi.puppet.com"}, - Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, + Git: Git{privateKey: ""}, + ForgeBaseURL: "https://forgeapi.puppet.com", + Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20, PurgeLevels: []string{"deployment"}, PurgeAllowList: []string{"custom.json", "**/*.xpp"}, DeploymentPurgeAllowList: []string{"full_hiera_*"}} @@ -695,7 +697,7 @@ func TestConfigUseCacheFallback(t *testing.T) { if msg, ok := err.(*exec.ExitError); ok { // there is error code exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() } - //fmt.Println(string(out)) + // fmt.Println(string(out)) if exitCode != 0 { t.Errorf("terminated with %v, but we expected exit status %v", exitCode, 0) @@ -708,6 +710,90 @@ func TestConfigUseCacheFallback(t *testing.T) { } } +func TestEnvFullSyncIfModuleWasTemporarilyNotAvailable(t *testing.T) { + funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] + config = readConfigfile(filepath.Join("tests", funcName+".yaml")) + branchParam = "single_git" + if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { + info = true + resolvePuppetEnvironment(false, "") + return + } + // be sure to delete files from previous test runs + gitDir := "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git" + purgeDir(gitDir, funcName) + purgeDir("/tmp/example/"+branchParam, funcName) + + // get the module to cache it + gm := GitModule{} + gm.git = "https://github.com/puppetlabs/puppetlabs-firewall.git" + doMirrorOrUpdate(gm, "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git", 0) + + // change the git remote url to something that does not resolve https://.com/... + er := executeCommand("git --git-dir "+gitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", 5, false) + if er.returnCode != 0 { + t.Error("Rewriting the git remote url of " + gitDir + " to https://.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output) + } + + cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") + cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") + out, err := cmd.CombinedOutput() + + exitCode := 0 + if msg, ok := err.(*exec.ExitError); ok { // there is error code + exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() + } + + expectedExitCode := 1 + if exitCode != expectedExitCode { + t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) + } + // fmt.Println(string(out)) + expectedLines := []string{ + "WARN: git repository https://github.com/puppetlabs/puppetlabs-firewall.git does not exist or is unreachable at this moment!", + "Fatal: Failed to clone or pull https://github.com/puppetlabs/puppetlabs-firewall.git to /tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git", + } + for _, expectedLine := range expectedLines { + if !strings.Contains(string(out), expectedLine) { + t.Errorf("Could not find expected line '" + expectedLine + "' in output") + } + } + // fix module again + er = executeCommand("git --git-dir "+gitDir+" remote set-url origin https://github.com/puppetlabs/puppetlabs-firewall.git", 5, false) + if er.returnCode != 0 { + t.Error("Rewriting the git remote url of " + gitDir + " to https://github.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output) + } + + // and do the sync again to check the output + cmdAgain := exec.Command(os.Args[0], "-test.run="+funcName+"$") + cmdAgain.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") + outAgain, err := cmdAgain.CombinedOutput() + exitCode = 0 + if msg, ok := err.(*exec.ExitError); ok { // there is error code + exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() + } + expectedExitCode = 0 + if exitCode != expectedExitCode { + t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) + } + + // fmt.Println("outAgain: ", string(outAgain)) + expectedLines = []string{ + "Need to sync /tmp/example/" + branchParam, + "Need to sync /tmp/example/" + branchParam + "/modules/firewall", + "Removing unmanaged path /tmp/example/" + branchParam + "/modules/foo", + } + for _, expectedLine := range expectedLines { + if !strings.Contains(string(outAgain), expectedLine) { + t.Errorf("Could not find expected line '" + expectedLine + "' in output") + } + } + + if !fileExists("/tmp/example/" + branchParam + "/modules/firewall/metadata.json") { + t.Errorf("terminated with the correct exit code and the correct output, but the resulting module was missing") + } +} + func TestConfigUseCacheFallbackFalse(t *testing.T) { quiet = true funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] @@ -750,7 +836,7 @@ func TestConfigUseCacheFallbackFalse(t *testing.T) { if exitCode != 1 { t.Errorf("terminated with %v, but we expected exit status %v", exitCode, 1) } - //fmt.Println(string(out)) + // fmt.Println(string(out)) if !strings.Contains(string(out), "WARN: git repository https://.com/puppetlabs/puppetlabs-firewall.git does not exist or is unreachable at this moment!") || !strings.Contains(string(out), "Fatal: Failed to clone or pull https://.com/puppetlabs/puppetlabs-firewall.git") { t.Errorf("terminated with the correct exit code, but the expected output was missing. out: %s", string(out)) } @@ -771,7 +857,7 @@ func TestReadPuppetfileUseCacheFallback(t *testing.T) { } purgeDir("/tmp/example", funcName) fm := ForgeModule{version: "1.9.0", author: "puppetlabs", name: "firewall"} - config.Forge.Baseurl = "https://forgeapi.puppet.com" + config.ForgeBaseURL = "https://forgeapi.puppet.com" downloadForgeModule("puppetlabs-firewall", "1.9.0", fm, 1) cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") @@ -1076,10 +1162,6 @@ func TestResolvePuppetfileFallback(t *testing.T) { if !fileExists(metadataFile) { t.Errorf("error missing file %s", metadataFile) } - - moduleParam = "" - debug = false - } func TestResolvePuppetfileDefaultBranch(t *testing.T) { @@ -1130,10 +1212,6 @@ func TestResolvePuppetfileDefaultBranch(t *testing.T) { if !fileExists(metadataFile) { t.Errorf("error missing file %s", metadataFile) } - - moduleParam = "" - debug = false - } func TestResolvePuppetfileControlBranch(t *testing.T) { @@ -1181,10 +1259,6 @@ func TestResolvePuppetfileControlBranch(t *testing.T) { if !fileExists(branchFile) { t.Errorf("error missing file %s, which means that not the correct module branch was used by :control_branch", branchFile) } - - moduleParam = "" - debug = false - } func TestResolvePuppetfileControlBranchDefault(t *testing.T) { @@ -1235,10 +1309,6 @@ func TestResolvePuppetfileControlBranchDefault(t *testing.T) { if !fileExists(metadataFile) { t.Errorf("error missing file %s", metadataFile) } - - moduleParam = "" - debug = false - } func TestConfigRetryGitCommands(t *testing.T) { @@ -1363,10 +1433,6 @@ func TestResolvePuppetfileLocalModules(t *testing.T) { if !fileExists(file2) { t.Errorf("error missing file %s", file2) } - - moduleParam = "" - debug = false - } func TestResolvePuppetfileInvalidGitObject(t *testing.T) { @@ -1398,10 +1464,6 @@ func TestResolvePuppetfileInvalidGitObject(t *testing.T) { if !strings.Contains(string(out), expectingString) { t.Errorf("terminated with the correct exit code, but the expected output was missing. out: %s\nExpecting string: %s", string(out), expectingString) } - - moduleParam = "" - debug = false - } func TestUnTarPreserveTimestamp(t *testing.T) { @@ -1501,10 +1563,6 @@ func TestSupportOldGitWithoutObjectSyntax(t *testing.T) { if !fileExists(metadataFile) { t.Errorf("error missing file %s", metadataFile) } - - moduleParam = "" - debug = false - } func TestSupportOldGitWithoutObjectSyntaxParameter(t *testing.T) { @@ -1556,10 +1614,6 @@ func TestSupportOldGitWithoutObjectSyntaxParameter(t *testing.T) { if !fileExists(metadataFile) { t.Errorf("error missing file %s", metadataFile) } - - moduleParam = "" - debug = false - } func TestAutoCorrectEnvironmentNamesDefault(t *testing.T) { @@ -1596,9 +1650,6 @@ func TestAutoCorrectEnvironmentNamesDefault(t *testing.T) { } purgeDir("/tmp/example", funcName) - moduleParam = "" - debug = false - } func TestAutoCorrectEnvironmentNamesWarn(t *testing.T) { @@ -1635,9 +1686,6 @@ func TestAutoCorrectEnvironmentNamesWarn(t *testing.T) { } purgeDir("/tmp/example", funcName) - moduleParam = "" - debug = false - } func TestAutoCorrectEnvironmentNamesError(t *testing.T) { @@ -1674,8 +1722,6 @@ func TestAutoCorrectEnvironmentNamesError(t *testing.T) { } purgeDir("/tmp/example", funcName) - moduleParam = "" - debug = false } func TestLastCheckedFile(t *testing.T) { @@ -1743,8 +1789,6 @@ func TestLastCheckedFile(t *testing.T) { purgeDir("/tmp/example", funcName) purgeDir("/tmp/g10k", funcName) - moduleParam = "" - debug = false } func TestSimplePostrunCommand(t *testing.T) { @@ -1782,8 +1826,6 @@ func TestSimplePostrunCommand(t *testing.T) { purgeDir("/tmp/example", funcName) purgeDir("/tmp/g10k", funcName) - moduleParam = "" - debug = false } func TestPostrunCommand(t *testing.T) { @@ -1833,8 +1875,6 @@ func TestPostrunCommand(t *testing.T) { purgeDir("/tmp/example", funcName) purgeDir("/tmp/g10k", funcName) - moduleParam = "" - debug = false } func TestPostrunCommandDirs(t *testing.T) { @@ -1886,8 +1926,6 @@ func TestPostrunCommandDirs(t *testing.T) { purgeDir("/tmp/example", funcName) purgeDir("/tmp/g10k", funcName) - moduleParam = "" - debug = false } func TestMultipleModuledirs(t *testing.T) { @@ -1939,11 +1977,9 @@ func TestMultipleModuledirs(t *testing.T) { t.Errorf("Unmanaged Module directory 2 is still there and should not be: %s", unmanagedModule2) } - //purgeDir("/tmp/example", funcName) + purgeDir("/tmp/example", funcName) purgeDir("/tmp/g10k", funcName) - moduleParam = "" branchParam = "" - debug = false } func TestFailedGit(t *testing.T) { @@ -2027,123 +2063,18 @@ func TestCheckDirPermissions(t *testing.T) { purgeDir("/tmp/example", funcName) } -func TestPurgeAllowList(t *testing.T) { +func TestPurgeStalePuppetfileOnly(t *testing.T) { funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] cacheDir := "/tmp/g10k" if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { debug = true - config = readConfigfile("tests/TestConfigExamplePurgeEnvironment.yaml") - branchParam = "single_git" + config = readConfigfile("tests/TestConfigFullworkingPurgePuppetfile.yaml") + branchParam = "" resolvePuppetEnvironment(false, "") - return - } - purgeDir("/tmp/example", funcName) - createOrPurgeDir("/tmp/example/single_git/stale_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/example/single_git/.resource_types", funcName) - f, _ := os.Create("/tmp/example/single_git/.latest_revision") - defer f.Close() - f.WriteString("foobar") - f.Sync() - frt, err := os.Create("/tmp/example/single_git/.resource_types/foobar.pp") - if err != nil { - t.Errorf("Error while creating test file") - } - defer frt.Close() - frt.WriteString("fake resource type") - frt.Sync() - createOrPurgeDir("/tmp/example/single_git/modules/", funcName) - createOrPurgeDir("/tmp/example/single_git/modules/firewall/", funcName) - createOrPurgeDir("/tmp/example/single_git/modules/firewall/manifests/", funcName) - fpf, _ := os.Create("/tmp/example/single_git/modules/firewall/manifests/stale.pp") - defer fpf.Close() - fpf.WriteString("fake stale module file") - fpf.Sync() - - cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") - cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") - out, err := cmd.CombinedOutput() - - exitCode := 0 - if msg, ok := err.(*exec.ExitError); ok { // there is error code - exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() - } - - expectedExitCode := 0 - if expectedExitCode != exitCode { - t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) - } - //fmt.Println(string(out)) - - expectedLines := []string{ - "Removing unmanaged path /tmp/example/single_git/stale_directory_that_should_be_purged", - "DEBUG purgeDir(): Trying to remove: /tmp/example/single_git/modules/firewall/manifests/stale.pp called from checkForStaleContent()", - "DEBUG purgeDir(): Trying to remove: /tmp/example/single_git/stale_directory_that_should_be_purged called from checkForStaleContent()", - } - - for _, expectedLine := range expectedLines { - if !strings.Contains(string(out), expectedLine) { - t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") - } - } - - expectedFiles := []string{ - "/tmp/example/single_git/.resource_types", - "/tmp/example/single_git/.resource_types/foobar.pp", - "/tmp/example/single_git/.latest_revision", - } - - for _, expectedFile := range expectedFiles { - if !fileExists(expectedFile) { - t.Errorf("purge_allowlist item was purged: " + expectedFile) - } - } - - missingFiles := []string{ - "/tmp/example/single_git/stale_directory_that_should_be_purged", - "/tmp/example/single_git/modules/firewall/manifests/stale.pp", - } - for _, missingFile := range missingFiles { - if fileExists(missingFile) { - t.Errorf("stale file and/or directory still exists! " + missingFile) - } - } - - if !fileExists("/tmp/example/single_git/modules/firewall/README.markdown") { - t.Errorf("Missing module file that should be there") - } - - purgeDir(cacheDir, funcName) - purgeDir("/tmp/example", funcName) -} - -func TestPurgeAllowListRecursive(t *testing.T) { - funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] - cacheDir := "/tmp/g10k" - if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { - debug = true - config = readConfigfile("tests/TestConfigExamplePurgeEnvironmentRecursive.yaml") - branchParam = "single_git" + createOrPurgeDir("/tmp/full/full_master/modules/stale_module_directory_that_should_be_purged", funcName) resolvePuppetEnvironment(false, "") return } - purgeDir("/tmp/example", funcName) - createOrPurgeDir("/tmp/example/single_git/stale_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/example/single_git/.resource_types", funcName) - f, _ := os.Create("/tmp/example/single_git/.latest_revision") - defer f.Close() - f.WriteString("foobar") - f.Sync() - frt, _ := os.Create("/tmp/example/single_git/.resource_types/foobar.pp") - defer frt.Close() - frt.WriteString("fake resource type") - frt.Sync() - createOrPurgeDir("/tmp/example/single_git/modules/", funcName) - createOrPurgeDir("/tmp/example/single_git/modules/firewall/", funcName) - createOrPurgeDir("/tmp/example/single_git/modules/firewall/manifests/", funcName) - fpf, _ := os.Create("/tmp/example/single_git/modules/firewall/manifests/stale.pp") - defer fpf.Close() - fpf.WriteString("fake stale module file") - fpf.Sync() cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") @@ -2158,11 +2089,11 @@ func TestPurgeAllowListRecursive(t *testing.T) { if expectedExitCode != exitCode { t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) } - //fmt.Println(string(out)) + // fmt.Println(string(out)) expectedLines := []string{ - "Removing unmanaged path /tmp/example/single_git/stale_directory_that_should_be_purged", - "DEBUG purgeDir(): Trying to remove: /tmp/example/single_git/stale_directory_that_should_be_purged called from checkForStaleContent()", + "DEBUG purgeDir(): Trying to remove: /tmp/full/full_master/modules/stale_module_directory_that_should_be_purged called from purge_level puppetfile", + "Removing unmanaged path /tmp/full/full_master/modules/stale_module_directory_that_should_be_purged", } for _, expectedLine := range expectedLines { @@ -2171,21 +2102,8 @@ func TestPurgeAllowListRecursive(t *testing.T) { } } - expectedFiles := []string{ - "/tmp/example/single_git/.resource_types", - "/tmp/example/single_git/.resource_types/foobar.pp", - "/tmp/example/single_git/.latest_revision", - "/tmp/example/single_git/modules/firewall/manifests/stale.pp", - } - - for _, expectedFile := range expectedFiles { - if !fileExists(expectedFile) { - t.Errorf("purge_allowlist item was purged: " + expectedFile) - } - } - missingFiles := []string{ - "/tmp/example/single_git/stale_directory_that_should_be_purged", + "/tmp/full/full_master/modules/stale_module_directory_that_should_be_purged", } for _, missingFile := range missingFiles { if fileExists(missingFile) { @@ -2193,35 +2111,30 @@ func TestPurgeAllowListRecursive(t *testing.T) { } } - if !fileExists("/tmp/example/single_git/modules/firewall/README.markdown") { + if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { t.Errorf("Missing module file that should be there") } purgeDir(cacheDir, funcName) - purgeDir("/tmp/example", funcName) + purgeDir("/tmp/full", funcName) } -func TestPurgeStaleContent(t *testing.T) { +func TestPurgeStaleDeploymentOnly(t *testing.T) { funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] cacheDir := "/tmp/g10k" if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { debug = true - config = readConfigfile("tests/TestConfigExamplePurgeEnvironment.yaml") - branchParam = "single" + config = readConfigfile("tests/TestConfigFullworkingPurgeDeployment.yaml") + branchParam = "" resolvePuppetEnvironment(false, "") return } - createOrPurgeDir("/tmp/example/single/stale_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/example/single/stale_directory_that_should_be_purged2", funcName) - f, _ := os.Create("/tmp/example/single/stale_directory_that_should_be_purged/stale_file") + createOrPurgeDir("/tmp/full/full_stale/stale_directory_that_should_be_purged", funcName) + createOrPurgeDir("/tmp/full/full_stale/stale_dir", funcName) + f, _ := os.Create("/tmp/full/full_stale/stale_dir/stale_file") defer f.Close() f.WriteString("foobar") f.Sync() - createOrPurgeDir("/tmp/example/single/.resource_types", funcName) - r, _ := os.Create("/tmp/example/single/.resource_types/test.pp") - defer r.Close() - r.WriteString("foobar") - r.Sync() cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") @@ -2239,13 +2152,15 @@ func TestPurgeStaleContent(t *testing.T) { // fmt.Println(string(out)) expectedLines := []string{ - "DEBUG checkForStaleContent(): filepath.Walk'ing directory /tmp/example/single", - "Removing unmanaged path /tmp/example/single/stale_directory_that_should_be_purged", - "DEBUG purgeDir(): Trying to remove: /tmp/example/single/stale_directory_that_should_be_purged called from checkForStaleContent()", - "Removing unmanaged path /tmp/example/single/stale_directory_that_should_be_purged/stale_file", - "DEBUG purgeDir(): Unnecessary to remove dir: /tmp/example/single/stale_directory_that_should_be_purged/stale_file it does not exist. Called from checkForStaleContent()", - "Removing unmanaged path /tmp/example/single/stale_directory_that_should_be_purged2", - "DEBUG purgeDir(): Trying to remove: /tmp/example/single/stale_directory_that_should_be_purged2 called from checkForStaleContent()", + "DEBUG purgeUnmanagedContent(): Glob'ing with path /tmp/full/full_*", + "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_another", + "DEBUG purgeUnmanagedContent(): Not purging environment full_another", + "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_master", + "DEBUG purgeUnmanagedContent(): Not purging environment full_master", + "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_qa", + "DEBUG purgeUnmanagedContent(): Not purging environment full_qa", + "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_stale", + "Removing unmanaged environment full_stale", } for _, expectedLine := range expectedLines { @@ -2254,35 +2169,36 @@ func TestPurgeStaleContent(t *testing.T) { } } - if fileExists("/tmp/example/single/stale_directory_that_should_be_purged/stale_file") || - fileExists("/tmp/example/single/stale_directory_that_should_be_purged") || - fileExists("/tmp/example/single/stale_directory_that_should_be_purged2") { + if fileExists("/tmp/full/full_stale/stale_directory_that_should_be_purged") || + fileExists("/tmp/full/full_stale/stale_dir") || + fileExists("/tmp/full/full_stale/stale_dir/stale_file") { t.Errorf("stale file and/or directory still exists!") } - if !fileExists("/tmp/example/single/external_modules/inifile/README.md") { + if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { t.Errorf("Missing module file that should be there") } - if !fileExists("/tmp/example/single/.resource_types/test.pp") { - t.Errorf("Missing resource_types file /tmp/example/single/.resource_types/test.pp that should be there") - } - purgeDir(cacheDir, funcName) - purgeDir("/tmp/example", funcName) + purgeDir("/tmp/full", funcName) } -func TestPurgeStaleEnvironments(t *testing.T) { +func TestPurgeStaleDeploymentOnlyWithAllowList(t *testing.T) { funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] cacheDir := "/tmp/g10k" if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { debug = true - config = readConfigfile("tests/TestConfigFullworking.yaml") + config = readConfigfile("tests/TestConfigFullworkingPurgeDeploymentWithAllowList.yaml") + branchParam = "" resolvePuppetEnvironment(false, "") return } + createOrPurgeDir("/tmp/full/full_master/modules/stale_module_directory_that_should_not_be_purged", funcName) + createOrPurgeDir("/tmp/full/full_master/stale_directory_that_should_not_be_purged", funcName) createOrPurgeDir("/tmp/full/full_stale/stale_directory_that_should_be_purged", funcName) createOrPurgeDir("/tmp/full/full_stale/stale_dir", funcName) + createOrPurgeDir("/tmp/full/full_hiera_master/hiera_dir", funcName) + createOrPurgeDir("/tmp/full/full_hiera_qa/hiera_dir_qa", funcName) f, _ := os.Create("/tmp/full/full_stale/stale_dir/stale_file") defer f.Close() f.WriteString("foobar") @@ -2327,6 +2243,16 @@ func TestPurgeStaleEnvironments(t *testing.T) { t.Errorf("stale file and/or directory still exists!") } + expectedFiles := []string{ + "/tmp/full/full_hiera_qa/hiera_dir_qa", + "/tmp/full/full_hiera_master/hiera_dir"} + + for _, expectedFile := range expectedFiles { + if !fileExists(expectedFile) { + t.Errorf("files and/or directory missing that should not have been purged! " + expectedFile) + } + } + if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { t.Errorf("Missing module file that should be there") } @@ -2335,25 +2261,17 @@ func TestPurgeStaleEnvironments(t *testing.T) { purgeDir("/tmp/full", funcName) } -func TestPurgeStaleEnvironmentOnly(t *testing.T) { +func TestEnvironmentParameter(t *testing.T) { funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] cacheDir := "/tmp/g10k" if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { debug = true - config = readConfigfile("tests/TestConfigFullworkingPurgeEnvironment.yaml") - fmt.Printf("%+v\n", config) + config = readConfigfile("tests/TestConfigFullworkingAndExampleDifferentPrefix.yaml") + environmentParam = "full_master" branchParam = "" resolvePuppetEnvironment(false, "") return } - createOrPurgeDir("/tmp/full/full_master/modules/stale_module_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_master/stale_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_dir", funcName) - f, _ := os.Create("/tmp/full/full_stale/stale_dir/stale_file") - defer f.Close() - f.WriteString("foobar") - f.Sync() cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") @@ -2371,8 +2289,8 @@ func TestPurgeStaleEnvironmentOnly(t *testing.T) { //fmt.Println(string(out)) expectedLines := []string{ - "Removing unmanaged path /tmp/full/full_master/stale_directory_that_should_not_be_purged", - "DEBUG purgeDir(): Trying to remove: /tmp/full/full_master/stale_directory_that_should_not_be_purged called from checkForStaleContent()", + "DEBUG 1(): Resolving environment master of source full", + "DEBUG resolvePuppetfile(): Resolving branch master of source full", } for _, expectedLine := range expectedLines { @@ -2381,298 +2299,8 @@ func TestPurgeStaleEnvironmentOnly(t *testing.T) { } } - missingFiles := []string{ - "/tmp/full/full_master/stale_directory_that_should_not_be_purged", - } - for _, missingFile := range missingFiles { - if fileExists(missingFile) { - t.Errorf("stale file and/or directory still exists! " + missingFile) - } - } - - expectedFiles := []string{ - "/tmp/full/full_stale/stale_directory_that_should_be_purged", - "/tmp/full/full_stale/stale_dir", - "/tmp/full/full_stale/stale_dir/stale_file", - } - for _, expectedFile := range expectedFiles { - if !fileExists(expectedFile) { - t.Errorf("stale files and/or directory missing that should not have been purged! " + expectedFile) - } - } - - if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { - t.Errorf("Missing module file that should be there") - } - - purgeDir(cacheDir, funcName) - purgeDir("/tmp/full", funcName) -} - -func TestPurgeStalePuppetfileOnly(t *testing.T) { - funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] - cacheDir := "/tmp/g10k" - if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { - debug = true - config = readConfigfile("tests/TestConfigFullworkingPurgePuppetfile.yaml") - branchParam = "" - resolvePuppetEnvironment(false, "") - return - } - createOrPurgeDir("/tmp/full/full_master/modules/stale_module_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_master/stale_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_dir", funcName) - f, _ := os.Create("/tmp/full/full_stale/stale_dir/stale_file") - defer f.Close() - f.WriteString("foobar") - f.Sync() - - cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") - cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") - out, err := cmd.CombinedOutput() - - exitCode := 0 - if msg, ok := err.(*exec.ExitError); ok { // there is error code - exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() - } - - expectedExitCode := 0 - if expectedExitCode != exitCode { - t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) - } - //fmt.Println(string(out)) - - expectedLines := []string{ - "Removing unmanaged path /tmp/full/full_master/modules/stale_module_directory_that_should_be_purged", - "DEBUG purgeDir(): Trying to remove: /tmp/full/full_master/modules/stale_module_directory_that_should_be_purged called from purge_level puppetfile", - } - - for _, expectedLine := range expectedLines { - if !strings.Contains(string(out), expectedLine) { - t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") - } - } - - missingFiles := []string{ - "/tmp/full/full_master/modules/stale_module_directory_that_should_be_purged", - } - for _, missingFile := range missingFiles { - if fileExists(missingFile) { - t.Errorf("stale file and/or directory still exists! " + missingFile) - } - } - - expectedFiles := []string{ - "/tmp/full/full_master/stale_directory_that_should_not_be_purged", - "/tmp/full/full_stale/stale_directory_that_should_not_be_purged", - "/tmp/full/full_stale/stale_dir", - "/tmp/full/full_stale/stale_dir/stale_file", - } - for _, expectedFile := range expectedFiles { - if !fileExists(expectedFile) { - t.Errorf("stale files and/or directory missing that should not have been purged! " + expectedFile) - } - } - - if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { - t.Errorf("Missing module file that should be there") - } - - purgeDir(cacheDir, funcName) - purgeDir("/tmp/full", funcName) -} - -func TestPurgeStaleDeploymentOnly(t *testing.T) { - funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] - cacheDir := "/tmp/g10k" - if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { - debug = true - config = readConfigfile("tests/TestConfigFullworkingPurgeDeployment.yaml") - branchParam = "" - resolvePuppetEnvironment(false, "") - return - } - createOrPurgeDir("/tmp/full/full_master/modules/stale_module_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_master/stale_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_dir", funcName) - f, _ := os.Create("/tmp/full/full_stale/stale_dir/stale_file") - defer f.Close() - f.WriteString("foobar") - f.Sync() - - cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") - cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") - out, err := cmd.CombinedOutput() - - exitCode := 0 - if msg, ok := err.(*exec.ExitError); ok { // there is error code - exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() - } - - expectedExitCode := 0 - if expectedExitCode != exitCode { - t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) - } - //fmt.Println(string(out)) - - expectedLines := []string{ - "DEBUG purgeUnmanagedContent(): Glob'ing with path /tmp/full/full_*", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_another", - "DEBUG purgeUnmanagedContent(): Not purging environment full_another", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_master", - "DEBUG purgeUnmanagedContent(): Not purging environment full_master", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_qa", - "DEBUG purgeUnmanagedContent(): Not purging environment full_qa", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_stale", - "Removing unmanaged environment full_stale", - } - - for _, expectedLine := range expectedLines { - if !strings.Contains(string(out), expectedLine) { - t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") - } - } - - if fileExists("/tmp/full/full_stale/stale_directory_that_should_be_purged") || - fileExists("/tmp/full/full_stale/stale_dir") || - fileExists("/tmp/full/full_stale/stale_dir/stale_file") { - t.Errorf("stale file and/or directory still exists!") - } - if !fileExists("/tmp/full/full_master/modules/stale_module_directory_that_should_not_be_purged") || - !fileExists("/tmp/full/full_master/stale_directory_that_should_not_be_purged") { - t.Errorf("stale files and/or directory missing that should not have been purged!") - } - - if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { - t.Errorf("Missing module file that should be there") - } - - purgeDir(cacheDir, funcName) - purgeDir("/tmp/full", funcName) -} - -func TestPurgeStaleDeploymentOnlyWithAllowList(t *testing.T) { - funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] - cacheDir := "/tmp/g10k" - if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { - debug = true - config = readConfigfile("tests/TestConfigFullworkingPurgeDeploymentWithAllowList.yaml") - branchParam = "" - resolvePuppetEnvironment(false, "") - return - } - createOrPurgeDir("/tmp/full/full_master/modules/stale_module_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_master/stale_directory_that_should_not_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_directory_that_should_be_purged", funcName) - createOrPurgeDir("/tmp/full/full_stale/stale_dir", funcName) - createOrPurgeDir("/tmp/full/full_hiera_master/hiera_dir", funcName) - createOrPurgeDir("/tmp/full/full_hiera_qa/hiera_dir_qa", funcName) - f, _ := os.Create("/tmp/full/full_stale/stale_dir/stale_file") - defer f.Close() - f.WriteString("foobar") - f.Sync() - - cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") - cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") - out, err := cmd.CombinedOutput() - - exitCode := 0 - if msg, ok := err.(*exec.ExitError); ok { // there is error code - exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() - } - - expectedExitCode := 0 - if expectedExitCode != exitCode { - t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) - } - //fmt.Println(string(out)) - - expectedLines := []string{ - "DEBUG purgeUnmanagedContent(): Glob'ing with path /tmp/full/full_*", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_another", - "DEBUG purgeUnmanagedContent(): Not purging environment full_another", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_master", - "DEBUG purgeUnmanagedContent(): Not purging environment full_master", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_qa", - "DEBUG purgeUnmanagedContent(): Not purging environment full_qa", - "DEBUG purgeUnmanagedContent(): Checking if environment should exist: full_stale", - "Removing unmanaged environment full_stale", - } - - for _, expectedLine := range expectedLines { - if !strings.Contains(string(out), expectedLine) { - t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") - } - } - - if fileExists("/tmp/full/full_stale/stale_directory_that_should_be_purged") || - fileExists("/tmp/full/full_stale/stale_dir") || - fileExists("/tmp/full/full_stale/stale_dir/stale_file") { - t.Errorf("stale file and/or directory still exists!") - } - - expectedFiles := []string{ - "/tmp/full/full_master/modules/stale_module_directory_that_should_not_be_purged", - "/tmp/full/full_master/stale_directory_that_should_not_be_purged", - "/tmp/full/full_hiera_qa/hiera_dir_qa", - "/tmp/full/full_hiera_master/hiera_dir"} - - for _, expectedFile := range expectedFiles { - if !fileExists(expectedFile) { - t.Errorf("stale files and/or directory missing that should not have been purged! " + expectedFile) - } - } - - if !fileExists("/tmp/full/full_master/modules/stdlib/metadata.json") { - t.Errorf("Missing module file that should be there") - } - - purgeDir(cacheDir, funcName) - purgeDir("/tmp/full", funcName) -} - -func TestEnvironmentParameter(t *testing.T) { - funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] - cacheDir := "/tmp/g10k" - if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { - debug = true - config = readConfigfile("tests/TestConfigFullworkingAndExampleDifferentPrefix.yaml") - environmentParam = "full_master" - branchParam = "" - resolvePuppetEnvironment(false, "") - return - } - - cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") - cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") - out, err := cmd.CombinedOutput() - - exitCode := 0 - if msg, ok := err.(*exec.ExitError); ok { // there is error code - exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() - } - - expectedExitCode := 0 - if expectedExitCode != exitCode { - t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) - } - //fmt.Println(string(out)) - - expectedLines := []string{ - "DEBUG 1(): Resolving environment master of source full", - "DEBUG resolvePuppetfile(): Resolving branch master of source full", - } - - for _, expectedLine := range expectedLines { - if !strings.Contains(string(out), expectedLine) { - t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") - } - } - - if fileExists("/tmp/out/example_master") { - t.Errorf("Puppet environment example_master should not have been deployed, with branch parameter set to full_master") + if fileExists("/tmp/out/example_master") { + t.Errorf("Puppet environment example_master should not have been deployed, with branch parameter set to full_master") } expectedFiles := []string{ @@ -2709,11 +2337,8 @@ func TestSkipPurgingWithMultipleSources(t *testing.T) { f, _ := os.Create("/tmp/out/example_single_git/mymodule2/dir1/file3") f.WriteString("slddkasjld") f.Close() - // do not manipulate a module locally, because r10k and g10k doesn't scan - // the module's directory content, if its version/commit hash didn't change - //f, _ = os.Create("/tmp/out/example_single_git/modules/firewall/unmanaged_file") - //f.WriteString("slddkasjld33") - //f.Close() + // and force another environment sync + purgeDir("/tmp/out/example_single_git/.g10k-deploy.json", funcName) f.Sync() branchParam = "" resolvePuppetEnvironment(false, "") @@ -2736,7 +2361,7 @@ func TestSkipPurgingWithMultipleSources(t *testing.T) { if expectedExitCode != exitCode { t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) } - //fmt.Println(string(out)) + // fmt.Println(string(out)) expectedLines := []string{ "Need to sync /tmp/out/example_single", @@ -3274,10 +2899,6 @@ func TestResolvePuppetfileUseSSHAgent(t *testing.T) { t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") } } - - moduleParam = "" - debug = false - } func TestResolvePuppetfileAutoDetectDefaultBranch(t *testing.T) { @@ -3308,3 +2929,154 @@ func TestResolvePuppetfileAutoDetectDefaultBranch(t *testing.T) { } //fmt.Println(string(out)) } + +func TestPrecedenceConfig(t *testing.T) { + funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] + config = readConfigfile("tests/TestConfigPuppetfilePrecedence.yaml") + if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { + debug = true + branchParam = "single_forge" + resolvePuppetEnvironment(false, "") + return + } + + cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") + cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") + out, err := cmd.CombinedOutput() + + exitCode := 0 + if msg, ok := err.(*exec.ExitError); ok { // there is error code + exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() + } + + expectedExitCode := 1 + if exitCode != expectedExitCode { + t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) + } + // fmt.Println(string(out)) + expectedLines := []string{ + "DEBUG resolveForgeModules(): Trying to get forge module puppetlabs/inifile-3.1.0 with Forge base url https://fake-forge.domain.tld and CacheTtl set to 100h0m0s", + } + + for _, expectedLine := range expectedLines { + if !strings.Contains(string(out), expectedLine) { + t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") + } + } +} + +func TestPrecedencePuppetfile(t *testing.T) { + funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] + config = readConfigfile("tests/TestConfigPuppetfilePrecedence.yaml") + if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { + debug = true + branchParam = "single_forge_precedence" + resolvePuppetEnvironment(false, "") + return + } + + cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") + cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") + out, err := cmd.CombinedOutput() + + exitCode := 0 + if msg, ok := err.(*exec.ExitError); ok { // there is error code + exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() + } + + expectedExitCode := 1 + if exitCode != expectedExitCode { + t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) + } + // fmt.Println(string(out)) + expectedLines := []string{ + "DEBUG resolveForgeModules(): Trying to get forge module puppetlabs/inifile-3.1.0 with Forge base url https://fake-forge-puppetfile.domain.tld and CacheTtl set to 24h1m1s", + } + + for _, expectedLine := range expectedLines { + if !strings.Contains(string(out), expectedLine) { + t.Errorf("Could not find expected line '" + expectedLine + "' in debug output") + } + } +} + +func TestPurgeControlRepoExceptModuledir(t *testing.T) { + funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1] + config = readConfigfile(filepath.Join("tests", "TestConfigUseCacheFallback.yaml")) + branchParam = "purge_control_repo_except_moduledir" + if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" { + debug = true + info = true + resolvePuppetEnvironment(false, "") + return + } + purgeDir("/tmp/example/", funcName) + + cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$") + cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") + out, err := cmd.CombinedOutput() + + exitCode := 0 + if msg, ok := err.(*exec.ExitError); ok { // there is error code + exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() + } + + expectedExitCode := 0 + if exitCode != expectedExitCode { + t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) + } + // fmt.Println(string(out)) + expectedLines := []string{ + "Need to sync /tmp/example/" + branchParam, + "Need to sync /tmp/example/" + branchParam + "/external_modules/inifile", + } + for _, expectedLine := range expectedLines { + if !strings.Contains(string(out), expectedLine) { + t.Errorf("Could not find expected line '" + expectedLine + "' in output") + } + } + + // force a resync of the Puppet env + purgeDir("/tmp/example/"+branchParam+"/.g10k-deploy.json", funcName) + + // and do the sync again to check if the modules dir was unncecessarily removed and repopulated + cmdAgain := exec.Command(os.Args[0], "-test.run="+funcName+"$") + cmdAgain.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1") + outAgain, err := cmdAgain.CombinedOutput() + exitCode = 0 + if msg, ok := err.(*exec.ExitError); ok { // there is error code + exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus() + } + expectedExitCode = 0 + if exitCode != expectedExitCode { + t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode) + } + + // fmt.Println("outAgain: ", string(outAgain)) + expectedLines = []string{ + "Need to sync /tmp/example/" + branchParam, + "Detected control repo change, but trying to preserve module dir /tmp/example/purge_control_repo_except_moduledir/external_modules", + "deleting /tmp/example/purge_control_repo_except_moduledir/Puppetfile", + "deleting /tmp/example/purge_control_repo_except_moduledir/bar", + "deleting /tmp/example/purge_control_repo_except_moduledir/foo", + } + for _, expectedLine := range expectedLines { + if !strings.Contains(string(outAgain), expectedLine) { + t.Errorf("Could not find expected line '" + expectedLine + "' in output") + } + } + + forbiddenLines := []string{ + "Need to sync /tmp/example/" + branchParam + "/external_modules/inifile", + "deleting /tmp/example/purge_control_repo_except_moduledir/foo/file001.bin", + } + for _, forbiddenLine := range forbiddenLines { + if strings.Contains(string(outAgain), forbiddenLine) { + t.Errorf("Found forbidden line '" + forbiddenLine + "' in output") + } + } + + if !fileExists("/tmp/example/" + branchParam + "/external_modules/inifile/metadata.json") { + t.Errorf("terminated with the correct exit code and the correct output, but the resulting module was missing") + } +} diff --git a/git.go b/git.go index 0aa86ad..7c61883 100644 --- a/git.go +++ b/git.go @@ -101,7 +101,7 @@ func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool explicitlyLoadSSHKey := true if len(gitModule.privateKey) == 0 || strings.Contains(gitModule.git, "github.com") || gitModule.useSSHAgent { - if gitModule.useSSHAgent { + if gitModule.useSSHAgent || len(gitModule.privateKey) == 0 { explicitlyLoadSSHKey = false } else if isControlRepo { explicitlyLoadSSHKey = true @@ -182,31 +182,18 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre if len(er.output) > 0 { commitHash := strings.TrimSuffix(er.output, "\n") if strings.HasPrefix(srcDir, config.EnvCacheDir) { - mutex.Lock() - desiredContent = append(desiredContent, deployFile) - mutex.Unlock() if fileExists(deployFile) { dr := readDeployResultFile(deployFile) - if dr.Signature == strings.TrimSuffix(er.output, "\n") { + if dr.Signature == strings.TrimSuffix(er.output, "\n") && dr.DeploySuccess { needToSync = false - // need to get the content of the git repository to detect and purge unmanaged files - addDesiredContent(srcDir, gitModule.tree, targetDir) } } } else { - Debugf("adding path to managed content: " + targetDir) - mutex.Lock() - desiredContent = append(desiredContent, hashFile) - desiredContent = append(desiredContent, targetDir) - mutex.Unlock() targetHashByte, _ := ioutil.ReadFile(hashFile) targetHash := string(targetHashByte) Debugf("string content of " + hashFile + " is: " + targetHash) if targetHash == commitHash { needToSync = false - mutex.Lock() - unchangedModuleDirs = append(unchangedModuleDirs, targetDir) - mutex.Unlock() Debugf("Skipping, because no diff found between " + srcDir + "(" + commitHash + ") and " + targetDir + "(" + targetHash + ")") } else { Debugf("Need to sync, because existing Git module: " + targetDir + " has commit " + targetHash + " and the to be synced commit is: " + commitHash) @@ -215,14 +202,47 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre } if needToSync && er.returnCode == 0 { - Infof("Need to sync " + targetDir) mutex.Lock() + Infof("Need to sync " + targetDir) needSyncDirs = append(needSyncDirs, targetDir) if _, ok := needSyncEnvs[correspondingPuppetEnvironment]; !ok { needSyncEnvs[correspondingPuppetEnvironment] = empty } needSyncGitCount++ mutex.Unlock() + moduleDir := "modules" + purgeWholeEnvDir := true + // check if it is a control repo and already exists + if isControlRepo && isDir(targetDir) { + // then check if it contains a Puppetfile + gitShowCmd := "git --git-dir " + srcDir + " show " + gitModule.tree + ":Puppetfile" + executeResult := executeCommand(gitShowCmd, config.Timeout, true) + Debugf("Executing " + gitShowCmd) + if executeResult.returnCode != 0 { + purgeWholeEnvDir = true + } else { + purgeWholeEnvDir = false + lines := strings.Split(executeResult.output, "\n") + for _, line := range lines { + if m := reModuledir.FindStringSubmatch(line); len(m) > 1 { + // moduledir CLI parameter override + if len(moduleDirParam) != 0 { + moduleDir = moduleDirParam + } else { + moduleDir = normalizeDir(m[1]) + } + } + } + } + } + // if so delete everything except the moduledir where the Puppet modules reside + // else simply delete the whole dir and check it out again + if purgeWholeEnvDir { + purgeDir(targetDir, "need to sync") + } else { + Infof("Detected control repo change, but trying to preserve module dir " + filepath.Join(targetDir, moduleDir)) + purgeControlRepoExceptModuledir(targetDir, moduleDir) + } if !dryRun && !config.CloneGitModules || isControlRepo { if pfMode { @@ -283,30 +303,6 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre return true } -// addDesiredContent takes the given git repository directory and the -// relevant reference (branch, commit hash, tag) and adds its content to -// the global desiredContent slice so that it doesn't get purged by g10k -func addDesiredContent(gitDir string, tree string, targetDir string) { - treeCmd := "git --git-dir " + gitDir + " ls-tree --full-tree -r -t --name-only " + tree - er := executeCommand(treeCmd, config.Timeout, false) - foundGitFiles := strings.Split(er.output, "\n") - mutex.Lock() - for _, desiredFile := range foundGitFiles[:len(foundGitFiles)-1] { - desiredContent = append(desiredContent, filepath.Join(targetDir, desiredFile)) - - // because we're using -r which prints git managed files in subfolders like this: foo/test3 - // we have to split up the given string and add the possible parent directories (foo in this case) - parentDirs := strings.Split(desiredFile, "/") - if len(parentDirs) > 1 { - for _, dir := range parentDirs[:len(parentDirs)-1] { - desiredContent = append(desiredContent, filepath.Join(targetDir, dir)) - } - } - } - mutex.Unlock() - -} - func detectDefaultBranch(gitDir string) string { remoteShowOriginCmd := "git ls-remote --symref " + gitDir er := executeCommand(remoteShowOriginCmd, config.Timeout, false) diff --git a/modules.go b/modules.go index 720f68f..3592470 100644 --- a/modules.go +++ b/modules.go @@ -38,9 +38,6 @@ func unTar(r io.Reader, targetBaseDir string) { continue } targetFilename := filepath.Join(targetBaseDir, filename) - mutex.Lock() - desiredContent = append(desiredContent, targetFilename) - mutex.Unlock() switch header.Typeflag { case tar.TypeDir: diff --git a/puppetfile.go b/puppetfile.go index e300d3c..629190e 100644 --- a/puppetfile.go +++ b/puppetfile.go @@ -35,7 +35,6 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) { allPuppetfiles := make(map[string]Puppetfile) allEnvironments := make(map[string]bool) allBasedirs := make(map[string]bool) - desiredContent = make([]string, 0) foundMatch := false for source, sa := range config.Sources { wg.Add() @@ -156,6 +155,16 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) { pf := filepath.Join(targetDir, "Puppetfile") if !fileExists(pf) { Debugf("resolvePuppetEnvironment(): Skipping branch " + source + "_" + branch + " because " + pf + " does not exist") + deployFile := filepath.Join(targetDir, ".g10k-deploy.json") + if fileExists(deployFile) { + Debugf("Finishing writing to deploy file " + deployFile) + dr := readDeployResultFile(deployFile) + dr.DeploySuccess = true + dr.FinishedAt = time.Now() + dr.GitDir = sa.Basedir + dr.GitURL = sa.Remote + writeStructJSONFile(deployFile, dr) + } } else { puppetfile := readPuppetfile(pf, sa.PrivateKey, source, branch, sa.ForceForgeVersions, false) puppetfile.workDir = normalizeDir(targetDir) @@ -164,7 +173,6 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) { puppetfile.gitURL = sa.Remote mutex.Lock() for _, moduleDir := range puppetfile.moduleDirs { - desiredContent = append(desiredContent, filepath.Join(puppetfile.workDir, moduleDir)) checkDirAndCreate(filepath.Join(puppetfile.workDir, moduleDir), "moduledir for env") } allPuppetfiles[env] = puppetfile @@ -250,9 +258,13 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) { continue } } - //fmt.Println("Found Forge module ", fm.author, "/", forgeModuleName, " with version", fm.version) fm.baseURL = pf.forgeBaseURL - fm.cacheTTL = pf.forgeCacheTTL + if pf.forgeCacheTTL != 0 { + fm.cacheTTL = pf.forgeCacheTTL + } else { + fm.cacheTTL = config.ForgeCacheTTL + } + // fmt.Println("Found Forge module", fm.author, "/", forgeModuleName, "with version", fm.version, "and cacheTTL", fm.cacheTTL) forgeModuleName = strings.Replace(forgeModuleName, "/", "-", -1) uniqueForgeModuleName := fm.author + "/" + forgeModuleName + "-" + fm.version if _, ok := uniqueForgeModules[uniqueForgeModuleName]; !ok { @@ -450,9 +462,6 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) { dr.GitDir = pf.gitDir dr.GitURL = pf.gitURL writeStructJSONFile(deployFile, dr) - mutex.Lock() - desiredContent = append(desiredContent, deployFile) - mutex.Unlock() } } diff --git a/stale.go b/stale.go index 38d1bb8..2dbb088 100644 --- a/stale.go +++ b/stale.go @@ -1,11 +1,8 @@ package main import ( - "os" "path/filepath" "strings" - - "github.com/yargevad/filepathx" ) func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[string]bool) { @@ -15,8 +12,8 @@ func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[stri return } } - for source, sa := range config.Sources { + // fmt.Printf("source: %+v\n", sa) prefix := resolveSourcePrefix(source, sa) if len(environmentParam) > 0 { @@ -52,11 +49,6 @@ func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[stri continue } } - if stringSliceContains(config.PurgeLevels, "environment") { - if allEnvironments[envName] { - checkForStaleContent(env) - } - } if stringSliceContains(config.PurgeLevels, "deployment") { Debugf("Checking if environment should exist: " + envName) if allEnvironments[envName] { @@ -72,57 +64,24 @@ func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[stri } } } - } else { - if stringSliceContains(config.PurgeLevels, "environment") { - // check for purgeable content inside -branch folder - checkForStaleContent(filepath.Join(sa.Basedir, prefix+branchParam)) - } } } } -func checkForStaleContent(workDir string) { - // add purge allowlist - if len(config.PurgeAllowList) > 0 { - for _, wlItem := range config.PurgeAllowList { - Debugf("interpreting purge allowlist globs: " + strings.Join(config.PurgeAllowList, " ")) - globPath := filepath.Join(workDir, wlItem) - Debugf("Glob'ing with purge allowlist glob " + globPath) - wlPaths, _ := filepathx.Glob(globPath) - Debugf("additional purge allowlist items: " + strings.Join(wlPaths, " ")) - desiredContent = append(desiredContent, wlPaths...) - } - } +func purgeControlRepoExceptModuledir(dir string, moduleDir string) { + moduleDir = filepath.Join(dir, moduleDir) - checkForStaleContent := func(path string, info os.FileInfo, err error) error { - stale := true - // never purge .resource_types dir and its content - if strings.HasSuffix(path, ".resource_types") || strings.HasSuffix(filepath.Dir(path), ".resource_types") { - stale = false + globPath := filepath.Join(dir, "*") + Debugf("Glob'ing with path " + globPath) + folders, _ := filepath.Glob(globPath) + for _, folder := range folders { + if folder == moduleDir || strings.HasPrefix(folder, moduleDir) { + continue } else { - for _, desiredFile := range desiredContent { - for _, unchangedModuleDir := range unchangedModuleDirs { - if strings.HasPrefix(path, unchangedModuleDir) { - stale = false - break - } - } - if path == desiredFile || path == workDir { - stale = false - break - } - } + Debugf("deleting " + folder) + purgeDir(folder, "purgeControlRepoExceptModuledir") } - if stale { - Debugf("Removing unmanaged path " + path) - purgeDir(path, "checkForStaleContent()") - } - return nil } - c := make(chan error) - Debugf("filepath.Walk'ing directory " + workDir) - go func() { c <- filepath.Walk(workDir, checkForStaleContent) }() - <-c // Walk done } diff --git a/tests/TestConfigPrefix.yaml b/tests/TestConfigPrefix.yaml index 553a67e..928500c 100644 --- a/tests/TestConfigPrefix.yaml +++ b/tests/TestConfigPrefix.yaml @@ -1,5 +1,6 @@ --- :cachedir: '/tmp/g10k' +forge_cache_ttl: 24h sources: example: diff --git a/tests/TestConfigPuppetfilePrecedence.yaml b/tests/TestConfigPuppetfilePrecedence.yaml new file mode 100644 index 0000000..cd08e84 --- /dev/null +++ b/tests/TestConfigPuppetfilePrecedence.yaml @@ -0,0 +1,9 @@ +--- +:cachedir: '/tmp/g10k' +forge_base_url: 'https://fake-forge.domain.tld' +forge_cache_ttl: 100h + +sources: + example: + remote: 'https://github.com/xorpaul/g10k-environment.git' + basedir: '/tmp/example/' diff --git a/tests/TestEnvFullSyncIfModuleWasTemporarilyNotAvailable.yaml b/tests/TestEnvFullSyncIfModuleWasTemporarilyNotAvailable.yaml new file mode 100644 index 0000000..fb73894 --- /dev/null +++ b/tests/TestEnvFullSyncIfModuleWasTemporarilyNotAvailable.yaml @@ -0,0 +1,8 @@ +--- +cachedir: '/tmp/g10k' +forge_cache_ttl: 24h + +sources: + example: + remote: 'https://github.com/xorpaul/g10k-environment.git' + basedir: '/tmp/example/'