clone branches to a special directory, to be able to clean it

api/internal/git/cloner.go

@@ -5,6 +5,7 @@ package git
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"sigs.k8s.io/kustomize/api/filesys"
 	"strings"
@@ -15,63 +16,92 @@ type Cloner func(repoSpec *RepoSpec) error
 
 func CachedGitCloner(repoSpec *RepoSpec) error {
 
+	var targetDir filesys.ConfirmedDir
 	gitRootDir, err := filesys.GitRootDir()
 
 	if err != nil {
 		return err
 	}
+	gitBranchesRootDir := gitRootDir.Join("branches")
 
 	repoFolderName := strings.ReplaceAll(fmt.Sprintf("%s_%s_%s", repoSpec.Host, repoSpec.OrgRepo, repoSpec.Ref), "/", "_")
-	targetDir := filesys.ConfirmedDir(gitRootDir.Join(repoFolderName))
 
-	if _, err := os.Stat(targetDir.String()); os.IsNotExist(err) {
+	permanentCachedDir := gitRootDir.Join(repoFolderName)
+	mutableCachedDir := filesys.ConfirmedDir(gitBranchesRootDir).Join(repoFolderName)
 
-		err := ClonerUsingGitExec(repoSpec)
-		if err != nil {
-			return err
-		}
+	//search into mutableCachedDir
+	if _, err := os.Stat(mutableCachedDir); os.IsNotExist(err) {
+		//search into immutable cached dir
+		if _, err := os.Stat(permanentCachedDir); os.IsNotExist(err) {
+			log.Printf("cloning %s", repoSpec.Raw())
+			err := ClonerUsingGitExec(repoSpec)
+			if err != nil {
+				return err
+			}
 
-		// check if it is a "tag" or simple branch
-		err = validateRepoRef(repoSpec)
-		if err != nil {
-			return err
-		}
+			// check if it is a "tag" a commitId or simple branch
+			isImmutableRef, err := isImmutableRepoRef(repoSpec)
+			if err != nil {
+				return err
+			}
+
+			if !isImmutableRef {
+				log.Printf("Git repo %v imported with a branch instead of tag or commitId", repoSpec.raw)
+				targetDir = filesys.ConfirmedDir(mutableCachedDir)
+			} else {
+				targetDir = filesys.ConfirmedDir(permanentCachedDir)
+			}
 
-		if _, err := os.Stat(gitRootDir.String()); os.IsNotExist(err) {
-			//create .kustomize dir if it does not exist
-			err = os.Mkdir(gitRootDir.String(), 0755|os.ModeDir)
+			if _, err := os.Stat(gitRootDir.String()); os.IsNotExist(err) {
+				//create .kustomize dir if it does not exist
+				err = os.Mkdir(gitRootDir.String(), 0755|os.ModeDir)
+				if err != nil {
+					return err
+				}
+			}
+			if _, err := os.Stat(gitBranchesRootDir); os.IsNotExist(err) {
+				//create .kustomize dir if it does not exist
+				err = os.Mkdir(gitBranchesRootDir, 0755|os.ModeDir)
+				if err != nil {
+					return err
+				}
+			}
+			log.Printf("move %s to %s", repoSpec.Dir.String(), targetDir.String())
+			err = os.Rename(repoSpec.Dir.String(), targetDir.String())
 			if err != nil {
 				return err
 			}
+
+		} else { // we found it and it is an immutable
+			targetDir = filesys.ConfirmedDir(permanentCachedDir)
 		}
-		err = os.Rename(repoSpec.Dir.String(), targetDir.String())
-		if err != nil {
-			return err
-		}
+	} else {
+		//we found it and it is an mutable
+		targetDir = filesys.ConfirmedDir(mutableCachedDir)
 
+		//TODO decide if we should clean old copy, update, or let the user clean the cache when he want
 	}
 
 	repoSpec.Dir = targetDir
 	return nil
 
 }
 
-func validateRepoRef(repoSpec *RepoSpec) error {
+// check if the git repo ref is not a banch
+func isImmutableRepoRef(repoSpec *RepoSpec) (bool, error) {
 	r, err := newCmdRunner()
 	r.dir = repoSpec.Dir
 	if err != nil {
-		return err
+		return false, err
 	}
-	err = r.run("fetch", "--tags")
+	origin, err := r.run("remote", "show", "origin")
 	if err != nil {
-		return err
+		return false, err
 	}
 
-	err = r.run("rev-parse", "-q", "--verify", repoSpec.Ref)
-	if err != nil {
-		return fmt.Errorf("Git repo %v imported with a branch instead of tag", repoSpec.raw)
-	}
-	return nil
+	remoteBranches := strings.Split(origin,"Remote branches:")[1]
+
+	return !strings.Contains(remoteBranches, "tracked"), nil
 }
 
 // ClonerUsingGitExec uses a local git install, as opposed
@@ -83,24 +113,27 @@ func ClonerUsingGitExec(repoSpec *RepoSpec) error {
 		return err
 	}
 	repoSpec.Dir = r.dir
-	if err = r.run("init"); err != nil {
+	if _, err = r.run("init"); err != nil {
 		return err
 	}
-	if err = r.run(
+	if _, err = r.run(
 		"remote", "add", "origin", repoSpec.CloneSpec()); err != nil {
 		return err
 	}
 	ref := "HEAD"
 	if repoSpec.Ref != "" {
 		ref = repoSpec.Ref
 	}
-	if err = r.run("fetch", "--depth=1", "origin", ref); err != nil {
+	if _, err = r.run("fetch", "--depth=1", "origin", ref); err != nil {
 		return err
 	}
-	if err = r.run("checkout", "FETCH_HEAD"); err != nil {
+	if _, err = r.run("checkout", "FETCH_HEAD"); err != nil {
 		return err
 	}
-	return r.run("submodule", "update", "--init", "--recursive")
+	if _, err = r.run("submodule", "update", "--init", "--recursive"); err != nil {
+		return err
+	}
+	return nil
 }
 
 // DoNothingCloner returns a cloner that only sets

api/internal/git/gitrunner.go

@@ -41,18 +41,22 @@ func newCmdRunner() (*gitRunner, error) {
 }
 
 // run a command with a timeout.
-func (r gitRunner) run(args ...string) error {
+func (r gitRunner) run(args ...string) (string, error) {
+	output := []byte{}
 	//nolint: gosec
 	cmd := exec.Command(r.gitProgram, args...)
 	cmd.Dir = r.dir.String()
-	return utils.TimedCall(
+
+	err := utils.TimedCall(
 		cmd.String(),
 		r.duration,
 		func() error {
-			_, err := cmd.CombinedOutput()
+			o, err := cmd.CombinedOutput()
 			if err != nil {
 				return errors.Wrapf(err, "git cmd = '%s'", cmd.String())
 			}
+			output = o
 			return err
 		})
+	return string(output), err
 }

api/loader/fileloader.go

@@ -178,7 +178,7 @@ func (fl *fileLoader) New(path string) (ifc.Loader, error) {
 			return nil, err
 		}
 		return newLoaderAtGitClone(
-			repoSpec, fl.fSys, fl, fl.cloner, fl.cleaner)
+			repoSpec, fl.fSys, fl, fl.cloner)
 	}
 
 	if filepath.IsAbs(path) {
@@ -202,7 +202,7 @@ func (fl *fileLoader) New(path string) (ifc.Loader, error) {
 // directory holding a cloned git repo.
 func newLoaderAtGitClone(
 	repoSpec *git.RepoSpec, fSys filesys.FileSystem,
-	referrer *fileLoader, cloner git.Cloner, cln func() error) (ifc.Loader, error) {
+	referrer *fileLoader, cloner git.Cloner) (ifc.Loader, error) {
 
 	cleaner := repoSpec.Cleaner(fSys)
 	err := cloner(repoSpec)
@@ -233,7 +233,7 @@ func newLoaderAtGitClone(
 		repoSpec:       repoSpec,
 		fSys:           fSys,
 		cloner:         cloner,
-		cleaner:        cln,
+		cleaner: 		func() error { return nil},
 	}, nil
 }
 

api/loader/fileloader_test.go

@@ -388,8 +388,7 @@ whatever
 	}
 	l, err := newLoaderAtGitClone(
 		repoSpec, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(coRoot)),
-		repoSpec.Cleaner(fSys))
+		git.DoNothingCloner(filesys.ConfirmedDir(coRoot)))
 	if err != nil {
 		t.Fatalf("unexpected err: %v\n", err)
 	}
@@ -468,8 +467,7 @@ func TestLoaderDisallowsLocalBaseFromRemoteOverlay(t *testing.T) {
 	}
 	l1, err = newLoaderAtGitClone(
 		repoSpec, fSys, nil,
-		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)),
-		repoSpec.Cleaner(fSys))
+		git.DoNothingCloner(filesys.ConfirmedDir(cloneRoot)))
 	if err != nil {
 		t.Fatalf("unexpected err: %v\n", err)
 	}

api/loader/loader.go

@@ -20,11 +20,10 @@ func NewLoader(
 	lr LoadRestrictorFunc,
 	target string, fSys filesys.FileSystem) (ifc.Loader, error) {
 	repoSpec, err := git.NewRepoSpecFromUrl(target)
-	dummyCleaner := func() error { return nil }
 	if err == nil {
 		// The target qualifies as a remote git target.
 		return newLoaderAtGitClone(
-			repoSpec, fSys, nil, git.CachedGitCloner, dummyCleaner)
+			repoSpec, fSys, nil, git.CachedGitCloner)
 	}
 	root, err := demandDirectoryRoot(fSys, target)
 	if err != nil {