remove stale detection (#199)

* add g10k config setting for forge base URL * remove stale content detection and add g10k config setting for forge base URL * remove stale content detection and add g10k config setting for forge base URL * remove stale content detection and add g10k config setting for forge base URL * remove stale content detection * remove stale content detection * remove stale content detection, add g10k config setting for forge base URL and fix deploy result file for branches containing no Puppetfile * remove stale content detection * add tests/TestConfigPuppetfilePrecedence.yaml to test for config setting precedence * add test for temporarily failing sync * fix test for TestConfigPrefix * fix test for github and only check output not exit code * debug test on github * fix TestPrecedence on github * prepare Puppetfile moduledir check * make reModuledir global to get moduledir from Puppetfile in syncToModuleDir() * do not unnecessarily purge moduledir of control repo * add purgeControlRepoExceptModuledir * add TestPurgeControlRepoExceptModuledir * list full path to Puppet env * also preserve content of module dir * use purge_control_repo_except_moduledir branch with dirs in control branch and check for delete lines in debug output * add delete debug output
xorpaul · Jun 23, 2022 · 864a119 · 864a119
1 parent 5a7d7dd
commit 864a119
Show file tree

Hide file tree

Showing 11 changed files with 427 additions and 675 deletions.
diff --git a/config.go b/config.go
@@ -14,6 +14,10 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+var (
+	reModuledir = regexp.MustCompile(`^\s*(?:moduledir)\s+['\"]?([^'\"]+)['\"]?`)
+)
+
 // readConfigfile creates the ConfigSettings struct from the g10k config file
 func readConfigfile(configFile string) ConfigSettings {
 	Debugf("Trying to read g10k config file: " + configFile)
@@ -51,11 +55,11 @@ func readConfigfile(configFile string) ConfigSettings {
 	config.ModulesCacheDir = checkDirAndCreate(filepath.Join(config.CacheDir, "modules"), "cachedir/modules")
 	config.EnvCacheDir = checkDirAndCreate(filepath.Join(config.CacheDir, "environments"), "cachedir/environments")
 
-	if len(config.Forge.Baseurl) == 0 {
-		config.Forge.Baseurl = "https://forgeapi.puppet.com"
+	if len(config.ForgeBaseURL) == 0 {
+		config.ForgeBaseURL = "https://forgeapi.puppet.com"
 	}
 
-	//fmt.Println("Forge Baseurl: ", config.Forge.Baseurl)
+	// fmt.Println("Forge Baseurl: ", config.ForgeBaseURL)
 
 	// set default timeout to 5 seconds if no timeout setting found
 	if config.Timeout == 0 {
@@ -98,6 +102,14 @@ func readConfigfile(configFile string) ConfigSettings {
 		config.MaxExtractworker = 20
 	}
 
+	if len(config.ForgeCacheTTLString) != 0 {
+		ttl, err := time.ParseDuration(config.ForgeCacheTTLString)
+		if err != nil {
+			Fatalf("Error: Can not convert value " + config.ForgeCacheTTLString + " of config setting forge_cache_ttl to a golang Duration. Valid time units are 300ms, 1.5h or 2h45m. In " + configFile)
+		}
+		config.ForgeCacheTTL = ttl
+	}
+
 	// check for non-empty config.Deploy which takes precedence over the non-deploy scoped settings
 	// See https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy
 	emptyDeploy := DeploySettings{}
@@ -132,7 +144,7 @@ func readConfigfile(configFile string) ConfigSettings {
 		Validatef()
 	}
 
-	//fmt.Printf("%+v\n", config)
+	// fmt.Printf("%+v\n", config)
 	return config
 }
 
@@ -190,7 +202,6 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc
 	}
 
 	reEmptyLine := regexp.MustCompile(`^\s*$`)
-	reModuledir := regexp.MustCompile(`^\s*(?:moduledir)\s+['\"]?([^'\"]+)['\"]?`)
 	reForgeCacheTTL := regexp.MustCompile(`^\s*(?:forge.cache(?:TTL|Ttl))\s+['\"]?([^'\"]+)['\"]?`)
 	reForgeBaseURL := regexp.MustCompile(`^\s*(?:forge.base(?:URL|Url))\s+['\"]?([^'\"]+)['\"]?`)
 	reForgeModule := regexp.MustCompile(`^\s*(?:mod)\s+['\"]?([^'\"]+[-/][^'\"]+)['\"](?:\s*)[,]?(.*)`)
@@ -302,6 +313,10 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc
 			if _, ok := puppetFile.gitModules[comp[1]]; ok {
 				Fatalf("Error: Forge Puppet module with same name found in " + pf + " for module " + comp[1] + " line: " + line)
 			}
+			// the base url in the Puppetfile takes precedence over an base url specified in the g10k config yaml
+			if len(puppetFile.forgeBaseURL) == 0 {
+				puppetFile.forgeBaseURL = config.ForgeBaseURL
+			}
 			puppetFile.forgeModules[comp[1]] = ForgeModule{version: forgeModuleVersion, name: comp[1], author: comp[0], sha256sum: forgeChecksum, moduleDir: moduleDir, sourceBranch: source + "_" + branch}
 		} else if m := reGitModule.FindStringSubmatch(line); len(m) > 1 {
 			gitModuleName := m[1]
@@ -431,6 +446,6 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc
 
 	puppetFile.moduleDirs = moduleDirs
 	puppetFile.sourceBranch = branch
-	//fmt.Printf("%+v\n", puppetFile)
+	// fmt.Printf("%+v\n", puppetFile)
 	return puppetFile
 }
diff --git a/forge.go b/forge.go
@@ -165,7 +165,7 @@ func doModuleInstallOrNothing(fm ForgeModule) {
 }
 
 func queryForgeAPI(fm ForgeModule) ForgeResult {
-	baseURL := config.Forge.Baseurl
+	baseURL := config.ForgeBaseURL
 	if len(fm.baseURL) > 0 {
 		baseURL = fm.baseURL
 	}
@@ -276,7 +276,7 @@ func parseForgeAPIResult(json string, fm ForgeModule) ForgeResult {
 
 // getMetadataForgeModule queries the configured Puppet Forge and return
 func getMetadataForgeModule(fm ForgeModule) ForgeModule {
-	baseURL := config.Forge.Baseurl
+	baseURL := config.ForgeBaseURL
 	if len(fm.baseURL) > 0 {
 		baseURL = fm.baseURL
 	}
@@ -361,7 +361,7 @@ func downloadForgeModule(name string, version string, fm ForgeModule, retryCount
 	fileName := name + "-" + version + ".tar.gz"
 
 	if !isDir(filepath.Join(config.ForgeCacheDir, name+"-"+version)) {
-		baseURL := config.Forge.Baseurl
+		baseURL := config.ForgeBaseURL
 		if len(fm.baseURL) > 0 {
 			baseURL = fm.baseURL
 		}
@@ -685,9 +685,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo
 				check4ForgeUpdate(m.name, me.version, latestForgeModules.m[moduleName])
 				latestForgeModules.RUnlock()
 			}
-			mutex.Lock()
-			unchangedModuleDirs = append(unchangedModuleDirs, targetDir)
-			mutex.Unlock()
 			return
 		}
 		// safe to do, because we ensured in doModuleInstallOrNothing() that -latest exists
@@ -714,9 +711,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo
 			}
 			if me.version == m.version {
 				Debugf("Nothing to do, existing Forge module: " + targetDir + " has the same version " + me.version + " as the to be synced version: " + m.version)
-				mutex.Lock()
-				unchangedModuleDirs = append(unchangedModuleDirs, targetDir)
-				mutex.Unlock()
 				return
 			}
 			Infof("Need to sync, because existing Forge module: " + targetDir + " has version " + me.version + " and the to be synced version is: " + m.version)
@@ -747,9 +741,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo
 	Infof("Need to sync " + targetDir)
 	if !dryRun {
 		targetDir = checkDirAndCreate(targetDir, "as targetDir for module "+name)
-		mutex.Lock()
-		desiredContent = append(desiredContent, targetDir)
-		mutex.Unlock()
 		var targetDirDevice, workDirDevice uint64
 		if fileInfo, err := os.Stat(targetDir); err == nil {
 			if fileInfo.Sys() != nil {
@@ -785,10 +776,6 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string, correspo
 			if err != nil {
 				Fatalf(funcName + "(): Can't make " + path + " relative to " + resolvedWorkDir + " Error: " + err.Error())
 			}
-			mutex.Lock()
-			desiredContent = append(desiredContent, filepath.Join(targetDir, target))
-			//Debugf("adding path to managed content: " + filepath.Join(targetDir, target))
-			mutex.Unlock()
 
 			if info.IsDir() {
 				if target != "." { // skip the root dir

diff --git a/g10k.go b/g10k.go
@@ -58,8 +58,6 @@ var (
 	maxworker                    int
 	maxExtractworker             int
 	forgeModuleDeprecationNotice string
-	desiredContent               []string
-	unchangedModuleDirs          []string
 )
 
 // LatestForgeModules contains a map of unique Forge modules
@@ -76,7 +74,6 @@ type ConfigSettings struct {
 	ModulesCacheDir             string
 	EnvCacheDir                 string
 	Git                         Git
-	Forge                       Forge
 	Sources                     map[string]Source
 	Timeout                     int            `yaml:"timeout"`
 	IgnoreUnreachableModules    bool           `yaml:"ignore_unreachable_modules"`
@@ -93,8 +90,11 @@ type ConfigSettings struct {
 	WriteLock                   string         `yaml:"write_lock"`
 	GenerateTypes               bool           `yaml:"generate_types"`
 	PuppetPath                  string         `yaml:"puppet_path"`
-	PurgeSkiplist              []string       `yaml:"purge_skiplist"`
+	PurgeSkiplist               []string       `yaml:"purge_skiplist"`
 	CloneGitModules             bool           `yaml:"clone_git_modules"`
+	ForgeBaseURL                string         `yaml:"forge_base_url"`
+	ForgeCacheTTLString         string         `yaml:"forge_cache_ttl"`
+	ForgeCacheTTL               time.Duration
 }
 
 // DeploySettings is a struct for settings for controlling how g10k deploys behave.
@@ -106,7 +106,7 @@ type DeploySettings struct {
 	WriteLock                string   `yaml:"write_lock"`
 	GenerateTypes            bool     `yaml:"generate_types"`
 	PuppetPath               string   `yaml:"puppet_path"`
-	PurgeSkiplist           []string `yaml:"purge_skiplist"`
+	PurgeSkiplist            []string `yaml:"purge_skiplist"`
 }
 
 // Forge is a simple struct that contains the base URL of
@@ -308,10 +308,9 @@ func main() {
 				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
 			}
 			// default purge_levels
-			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppet.com"}
 			modulesCacheDir := filepath.Join(cachedir, "modules")
 			envsCacheDir := filepath.Join(cachedir, "environments")
-			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: modulesCacheDir, EnvCacheDir: envsCacheDir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
+			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: modulesCacheDir, EnvCacheDir: envsCacheDir, Sources: sm, ForgeBaseURL: "https://forgeapi.puppet.com", Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
 			config.PurgeLevels = []string{"puppetfile"}
 			target = pfLocation
 			puppetfile := readPuppetfile(target, "", "cmdlineparam", "cmdlineparam", false, false)