Skip to content

Commit

Permalink
add support for NO_PROXY (#217)
Browse files Browse the repository at this point in the history 
* add support for NO_PROXY environment variable where a http proxy should not be used for git Puppet modules
  • Loading branch information
@xorpaul
xorpaul authored May 25, 2023
1 parent 8d600b8 commit 6e075ed
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 17 deletions.
47 changes: 42 additions & 5 deletions g10k_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -684,7 +684,7 @@ func TestConfigUseCacheFallback(t *testing.T) {
}

// change the git remote url to something that does not resolve https://.com/...
er := executeCommand("git --git-dir "+unresolvableGitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false)
er := executeCommand("git --git-dir "+unresolvableGitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false, false)
if er.returnCode != 0 {
t.Error("Rewriting the git remote url of " + unresolvableGitDir + " to https://.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output)
}
Expand Down Expand Up @@ -730,7 +730,7 @@ func TestEnvFullSyncIfModuleWasTemporarilyNotAvailable(t *testing.T) {
doMirrorOrUpdate(gm, "/tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git", 0)

// change the git remote url to something that does not resolve https://.com/...
er := executeCommand("git --git-dir "+gitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false)
er := executeCommand("git --git-dir "+gitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false, false)
if er.returnCode != 0 {
t.Error("Rewriting the git remote url of " + gitDir + " to https://.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output)
}
Expand Down Expand Up @@ -759,7 +759,7 @@ func TestEnvFullSyncIfModuleWasTemporarilyNotAvailable(t *testing.T) {
}
}
// fix module again
er = executeCommand("git --git-dir "+gitDir+" remote set-url origin https://github.com/puppetlabs/puppetlabs-firewall.git", "", 5, false)
er = executeCommand("git --git-dir "+gitDir+" remote set-url origin https://github.com/puppetlabs/puppetlabs-firewall.git", "", 5, false, false)
if er.returnCode != 0 {
t.Error("Rewriting the git remote url of " + gitDir + " to https://github.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output)
}
Expand Down Expand Up @@ -819,7 +819,7 @@ func TestConfigUseCacheFallbackFalse(t *testing.T) {
}

// change the git remote url to something that does not resolve https://.com/...
er := executeCommand("git --git-dir "+unresolvableGitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false)
er := executeCommand("git --git-dir "+unresolvableGitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false, false)
if er.returnCode != 0 {
t.Error("Rewriting the git remote url of " + unresolvableGitDir + " to https://.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output)
}
Expand Down Expand Up @@ -2000,7 +2000,7 @@ func TestFailedGit(t *testing.T) {
doMirrorOrUpdate(gm, gitDir, 0)

// change the git remote url to something that does not resolve https://.com/...
er := executeCommand("git --git-dir "+gitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false)
er := executeCommand("git --git-dir "+gitDir+" remote set-url origin https://.com/puppetlabs/puppetlabs-firewall.git", "", 5, false, false)
if er.returnCode != 0 {
t.Error("Rewriting the git remote url of " + gitDir + " to https://.com/puppetlabs/puppetlabs-firewall.git failed! Errorcode: " + strconv.Itoa(er.returnCode) + "Error: " + er.output)
}
Expand Down Expand Up @@ -3225,3 +3225,40 @@ func TestStripComponentConflict(t *testing.T) {
}
}
}

func TestNoProxy(t *testing.T) {
funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
config = readConfigfile(filepath.Join("tests", "TestConfigPrefix.yaml"))
branchParam = "no_proxy"
if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
debug = true
info = true
resolvePuppetEnvironment(false, "")
return
}
purgeDir("/tmp/example/", funcName)

cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$")
cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1", "NO_PROXY=https://localgit.domain.tld")
out, err := cmd.CombinedOutput()

exitCode := 0
if msg, ok := err.(*exec.ExitError); ok { // there is error code
exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus()
}

expectedExitCode := 1
if exitCode != expectedExitCode {
t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode)
}
// fmt.Println(string(out))

expectedLines := []string{
"found matching NO_PROXY URL, trying to disable http_proxy for git clone --mirror https://localgit.domain.tld/foo/bar.git /tmp/g10k/modules/https-__localgit.domain.tld_foo_bar.git",
}
for _, expectedLine := range expectedLines {
if !strings.Contains(string(out), expectedLine) {
t.Errorf("Could not find expected line '" + expectedLine + "' in output")
}
}
}
20 changes: 13 additions & 7 deletions git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,20 @@ func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool
}
}

// check if git URL does match NO_PROXY
disableHttpProxy := false
if matchGitRemoteURLNoProxy(gitModule.git) {
disableHttpProxy = true
}

if explicitlyLoadSSHKey {
sshAddCmd := "ssh-add "
if runtime.GOOS == "darwin" {
sshAddCmd = "ssh-add -K "
}
er = executeCommand("ssh-agent bash -c '"+sshAddCmd+gitModule.privateKey+"; "+gitCmd+"'", "", config.Timeout, gitModule.ignoreUnreachable)
er = executeCommand("ssh-agent bash -c '"+sshAddCmd+gitModule.privateKey+"; "+gitCmd+"'", "", config.Timeout, gitModule.ignoreUnreachable, disableHttpProxy)
} else {
er = executeCommand(gitCmd, "", config.Timeout, gitModule.ignoreUnreachable)
er = executeCommand(gitCmd, "", config.Timeout, gitModule.ignoreUnreachable, disableHttpProxy)
}

if er.returnCode != 0 {
Expand All @@ -151,7 +157,7 @@ func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool
if config.CloneGitModules && !isControlRepo && !isInModulesCacheDir {
// if clone of git modules was specified, switch to the module and try to switch to the reference commit hash/tag/branch
gitCmd = "git checkout " + gitModule.tree
er = executeCommand(gitCmd, workDir, config.Timeout, gitModule.ignoreUnreachable)
er = executeCommand(gitCmd, workDir, config.Timeout, gitModule.ignoreUnreachable, disableHttpProxy)
if er.returnCode != 0 {
Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment! Error: " + er.output)
return false
Expand Down Expand Up @@ -180,7 +186,7 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre

isControlRepo := strings.HasPrefix(srcDir, config.EnvCacheDir)

er := executeCommand(revParseCmd, "", config.Timeout, gitModule.ignoreUnreachable)
er := executeCommand(revParseCmd, "", config.Timeout, gitModule.ignoreUnreachable, false)
hashFile := filepath.Join(targetDir, ".latest_commit")
deployFile := filepath.Join(targetDir, ".g10k-deploy.json")
needToSync := true
Expand Down Expand Up @@ -229,7 +235,7 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre
if isControlRepo && isDir(targetDir) {
// then check if it contains a Puppetfile
gitShowCmd := "git --git-dir " + srcDir + " show " + gitModule.tree + ":Puppetfile"
executeResult := executeCommand(gitShowCmd, "", config.Timeout, true)
executeResult := executeCommand(gitShowCmd, "", config.Timeout, true, false)
Debugf("Executing " + gitShowCmd)
if executeResult.returnCode != 0 {
purgeWholeEnvDir = true
Expand Down Expand Up @@ -318,7 +324,7 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre

func detectDefaultBranch(gitDir string) string {
remoteShowOriginCmd := "git ls-remote --symref " + gitDir
er := executeCommand(remoteShowOriginCmd, "", config.Timeout, false)
er := executeCommand(remoteShowOriginCmd, "", config.Timeout, false, false)
foundRefs := strings.Split(er.output, "\n")
if len(foundRefs) < 1 {
Fatalf("Unable to detect default branch for git repository with command git ls-remote --symref " + gitDir)
Expand All @@ -334,7 +340,7 @@ func detectDefaultBranch(gitDir string) string {
func detectGitRemoteURLChange(d string, url string) bool {
gitRemoteCmd := "git --git-dir " + d + " remote -v"

er := executeCommand(gitRemoteCmd, "", config.Timeout, false)
er := executeCommand(gitRemoteCmd, "", config.Timeout, false, false)
if er.returnCode != 0 {
Warnf("WARN: Could not detect remote URL for git repository " + d + " trying to purge it and mirror it again")
return true
Expand Down
32 changes: 30 additions & 2 deletions helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ func purgeDir(dir string, callingFunction string) {
}
}

func executeCommand(command string, commandDir string, timeout int, allowFail bool) ExecResult {
func executeCommand(command string, commandDir string, timeout int, allowFail bool, disableHttpProxy bool) ExecResult {
if len(commandDir) > 0 {
Debugf("Executing " + command + " in cwd " + commandDir)
} else {
Expand All @@ -193,6 +193,11 @@ func executeCommand(command string, commandDir string, timeout int, allowFail bo
if len(commandDir) > 0 {
execCommand.Dir = commandDir
}
if disableHttpProxy {
Debugf("found matching NO_PROXY URL, trying to disable http_proxy for " + command)
execCommand.Env = append(os.Environ(), "http_proxy=")
execCommand.Env = append(os.Environ(), "https_proxy=")
}
out, err := execCommand.CombinedOutput()
duration := time.Since(before).Seconds()
er := ExecResult{0, string(out)}
Expand Down Expand Up @@ -241,7 +246,7 @@ func checkForAndExecutePostrunCommand() {
postrunCommandString = strings.Replace(postrunCommandString, "$modifiedenvs", needSyncEnvText, -1)
postrunCommandString = strings.Replace(postrunCommandString, "$branchparam", branchParam, -1)

er := executeCommand(postrunCommandString, "", config.Timeout, false)
er := executeCommand(postrunCommandString, "", config.Timeout, false, false)
Debugf("postrun command '" + postrunCommandString + "' terminated with exit code " + strconv.Itoa(er.returnCode))
}
}
Expand Down Expand Up @@ -340,3 +345,26 @@ func stripComponent(component string, env string) string {
return strings.TrimPrefix(env, component)
}
}

func matchGitRemoteURLNoProxy(url string) bool {
noProxy := os.Getenv("NO_PROXY")
for _, np := range strings.Split(noProxy, ",") {
if len(np) > 0 {
if strings.Contains(url, np) {
// fmt.Println("found matching", np, "for", url)
return true
}
}
}
// do the same for lower case environment variable name
noProxyL := os.Getenv("no_proxy")
for _, np := range strings.Split(noProxyL, ",") {
if len(np) > 0 {
if strings.Contains(url, np) {
// fmt.Println("found matching", np, "for", url)
return true
}
}
}
return false
}
6 changes: 3 additions & 3 deletions puppetfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) {
if success := doMirrorOrUpdate(controlRepoGit, workDir, 0); success {

// get all branches
er := executeCommand("git --git-dir "+workDir+" branch", "", config.Timeout, false)
er := executeCommand("git --git-dir "+workDir+" branch", "", config.Timeout, false, false)
outputBranches := er.output
outputTags := ""

if tags {
er := executeCommand("git --git-dir "+workDir+" tag", "", config.Timeout, false)
er := executeCommand("git --git-dir "+workDir+" tag", "", config.Timeout, false, false)
outputTags = er.output
}

Expand Down Expand Up @@ -489,7 +489,7 @@ func skipBasedOnFilterCommand(branch string, sourceName string, sa Source, workD
branchFilterCommand = strings.ReplaceAll(branchFilterCommand, "$G10K_NAME", sourceName)
branchFilterCommand = strings.ReplaceAll(branchFilterCommand, "$GIT_DIR", workDir)
Debugf("executing filter_command: " + branchFilterCommand)
er := executeCommand(branchFilterCommand, "", 30, true)
er := executeCommand(branchFilterCommand, "", 30, true, false)
if debug {
fmt.Printf("filter_command %s result: %+v", branchFilterCommand, er)
}
Expand Down
7 changes: 7 additions & 0 deletions tests/TestNoProxyPuppetfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
mod 'puppetlabs/apt', 'latest'
mod 'bar',
:git => 'https://localgit.domain.tld/foo/bar.git'

mod 'firewall',
:git => 'https://github.com/puppetlabs/puppetlabs-firewall.git',
:tag => '1.15.3'

0 comments on commit 6e075ed

Please sign in to comment.