feat: git sparse checkout

Uses includePaths and excludePaths to configure non-cone sparse checkout for a git repo. Fixes carvel-dev#400 Signed-off-by: Zoltán Reegn <zoltan.reegn@gmail.com>
reegnz · Oct 25, 2024 · ce695e9 · ce695e9
1 parent 09ab9ab
commit ce695e9
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 36 deletions.
diff --git a/pkg/vendir/config/config.go b/pkg/vendir/config/config.go
@@ -178,13 +178,15 @@ func (c Config) UseDirectory(path, dirPath string) error {
 			matched = true
 
 			newCon := DirectoryContents{
-				Path:         con.Path,
-				Directory:    &DirectoryContentsDirectory{Path: dirPath},
-				IncludePaths: con.IncludePaths,
-				ExcludePaths: con.ExcludePaths,
-				IgnorePaths:  con.IgnorePaths,
-				LegalPaths:   con.LegalPaths,
-				Lazy:         con.Lazy,
+				Path:      con.Path,
+				Directory: &DirectoryContentsDirectory{Path: dirPath},
+				ContentPaths: ContentPaths{
+					IncludePaths: con.IncludePaths,
+					ExcludePaths: con.ExcludePaths,
+					IgnorePaths:  con.IgnorePaths,
+				},
+				LegalPaths: con.LegalPaths,
+				Lazy:       con.Lazy,
 			}
 			dir.Contents[j] = newCon
 			c.Directories[i] = dir

diff --git a/pkg/vendir/config/directory.go b/pkg/vendir/config/directory.go
@@ -46,17 +46,18 @@ type DirectoryContents struct {
 	Manual        *DirectoryContentsManual        `json:"manual,omitempty"`
 	Directory     *DirectoryContentsDirectory     `json:"directory,omitempty"`
 	Inline        *DirectoryContentsInline        `json:"inline,omitempty"`
-
-	IncludePaths []string `json:"includePaths,omitempty"`
-	ExcludePaths []string `json:"excludePaths,omitempty"`
-	IgnorePaths  []string `json:"ignorePaths,omitempty"`
+	Permissions   *os.FileMode                    `json:"permissions,omitempty"`
+	NewRootPath   string                          `json:"newRootPath,omitempty"`
+	ContentPaths  `json:",inline"`
 
 	// By default LICENSE/LICENCE/NOTICE/COPYRIGHT files are kept
 	LegalPaths *[]string `json:"legalPaths,omitempty"`
+}
 
-	NewRootPath string `json:"newRootPath,omitempty"`
-
-	Permissions *os.FileMode `json:"permissions,omitempty"`
+type ContentPaths struct {
+	IncludePaths []string `json:"includePaths,omitempty"`
+	ExcludePaths []string `json:"excludePaths,omitempty"`
+	IgnorePaths  []string `json:"ignorePaths,omitempty"`
 }
 
 type DirectoryContentsGit struct {
@@ -73,6 +74,7 @@ type DirectoryContentsGit struct {
 	SkipInitSubmodules     bool `json:"skipInitSubmodules,omitempty"`
 	Depth                  int  `json:"depth,omitempty"`
 	ForceHTTPBasicAuth     bool `json:"forceHTTPBasicAuth,omitempty"`
+	SparseCheckout         bool `json:"sparseCheckout,omitempty"`
 }
 
 type DirectoryContentsGitVerification struct {

diff --git a/pkg/vendir/directory/directory.go b/pkg/vendir/directory/directory.go
@@ -108,7 +108,8 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) {
 
 		switch {
 		case contents.Git != nil:
-			gitSync := ctlgit.NewSync(*contents.Git, NewInfoLog(d.ui), syncOpts.RefFetcher, syncOpts.Cache)
+			contentPaths := contents.ContentPaths
+			gitSync := ctlgit.NewSync(*contents.Git, contentPaths, NewInfoLog(d.ui), syncOpts.RefFetcher, syncOpts.Cache)
 
 			d.ui.PrintLinef("Fetching: %s + %s (git from %s)", d.opts.Path, contents.Path, gitSync.Desc())
 

diff --git a/pkg/vendir/fetch/git/git.go b/pkg/vendir/fetch/git/git.go
@@ -21,23 +21,24 @@ import (
 )
 
 type Git struct {
-	opts       ctlconf.DirectoryContentsGit
-	infoLog    io.Writer
-	refFetcher ctlfetch.RefFetcher
-	cmdRunner  CommandRunner
+	opts         ctlconf.DirectoryContentsGit
+	contentPaths ctlconf.ContentPaths
+	infoLog      io.Writer
+	refFetcher   ctlfetch.RefFetcher
+	cmdRunner    CommandRunner
 }
 
-func NewGit(opts ctlconf.DirectoryContentsGit,
-	infoLog io.Writer, refFetcher ctlfetch.RefFetcher) *Git {
-
-	return &Git{opts, infoLog, refFetcher, &runner{infoLog}}
+func NewGit(opts ctlconf.DirectoryContentsGit, contentPaths ctlconf.ContentPaths,
+	infoLog io.Writer, refFetcher ctlfetch.RefFetcher,
+) *Git {
+	return &Git{opts, contentPaths, infoLog, refFetcher, &runner{infoLog}}
 }
 
 // NewGitWithRunner creates a Git retriever with a provided runner
-func NewGitWithRunner(opts ctlconf.DirectoryContentsGit,
-	infoLog io.Writer, refFetcher ctlfetch.RefFetcher, cmdRunner CommandRunner) *Git {
-
-	return &Git{opts, infoLog, refFetcher, cmdRunner}
+func NewGitWithRunner(opts ctlconf.DirectoryContentsGit, contentPaths ctlconf.ContentPaths,
+	infoLog io.Writer, refFetcher ctlfetch.RefFetcher, cmdRunner CommandRunner,
+) *Git {
+	return &Git{opts, contentPaths, infoLog, refFetcher, cmdRunner}
 }
 
 //nolint:revive
@@ -142,6 +143,19 @@ func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea, bundle string) e
 		{"remote", "add", "origin", gitURL},
 	}
 
+	if t.opts.SparseCheckout {
+		if len(t.contentPaths.IncludePaths) > 0 || len(t.contentPaths.ExcludePaths) > 0 {
+			sparseCheckoutArgs := []string{"sparse-checkout", "set", "--no-cone"}
+			for _, include := range t.contentPaths.IncludePaths {
+				sparseCheckoutArgs = append(sparseCheckoutArgs, include)
+			}
+			for _, exclude := range t.contentPaths.ExcludePaths {
+				sparseCheckoutArgs = append(sparseCheckoutArgs, fmt.Sprintf("!%s", exclude))
+			}
+			argss = append(argss, sparseCheckoutArgs)
+		}
+	}
+
 	if authOpts.Username != nil && authOpts.Password != nil {
 		if !strings.HasPrefix(gitURL, "https://") {
 			return fmt.Errorf("Username/password authentication is only supported for https remotes")

diff --git a/pkg/vendir/fetch/git/sync.go b/pkg/vendir/fetch/git/sync.go
@@ -19,16 +19,17 @@ import (
 const gitCacheType = "git-bundle"
 
 type Sync struct {
-	opts       ctlconf.DirectoryContentsGit
-	log        io.Writer
-	refFetcher ctlfetch.RefFetcher
-	cache      ctlcache.Cache
+	opts         ctlconf.DirectoryContentsGit
+	contentPaths ctlconf.ContentPaths
+	log          io.Writer
+	refFetcher   ctlfetch.RefFetcher
+	cache        ctlcache.Cache
 }
 
-func NewSync(opts ctlconf.DirectoryContentsGit,
-	log io.Writer, refFetcher ctlfetch.RefFetcher, cache ctlcache.Cache) Sync {
-
-	return Sync{opts, log, refFetcher, cache}
+func NewSync(opts ctlconf.DirectoryContentsGit, contentPaths ctlconf.ContentPaths,
+	log io.Writer, refFetcher ctlfetch.RefFetcher, cache ctlcache.Cache,
+) Sync {
+	return Sync{opts, contentPaths, log, refFetcher, cache}
 }
 
 func (d Sync) Desc() string {
@@ -54,7 +55,7 @@ func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDire
 
 	cacheID := fmt.Sprintf("%x", sha256.Sum256([]byte(d.opts.URL)))
 
-	git := NewGit(d.opts, d.log, d.refFetcher)
+	git := NewGit(d.opts, d.contentPaths, d.log, d.refFetcher)
 
 	var bundle string
 	if cacheEntry, hasCache := d.cache.Has(gitCacheType, cacheID); hasCache {