From ccd20941b253c692cb8004b61a6e55d39aa7949e Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Wed, 31 Jul 2024 17:02:42 +0200 Subject: [PATCH] refactor: git package to be thread safe and not depend on types package Signed-off-by: Philip Laine --- go.mod | 4 +- go.sum | 4 + src/extensions/bigbang/bigbang.go | 2 +- src/internal/git/fallback.go | 58 ++++++ src/internal/git/git.go | 26 +++ src/internal/git/git_test.go | 40 ++++ src/internal/git/repository.go | 268 ++++++++++++++++++++++++++ src/internal/git/repository_test.go | 85 ++++++++ src/internal/packager/git/checkout.go | 97 ---------- src/internal/packager/git/clone.go | 126 ------------ src/internal/packager/git/common.go | 54 ------ src/internal/packager/git/pull.go | 81 -------- src/internal/packager/git/push.go | 143 -------------- src/internal/packager/helm/repo.go | 22 +-- src/pkg/packager/creator/normal.go | 6 +- src/pkg/packager/deploy.go | 20 +- src/pkg/packager/filters/diff.go | 2 +- 17 files changed, 511 insertions(+), 527 deletions(-) create mode 100644 src/internal/git/fallback.go create mode 100644 src/internal/git/git.go create mode 100644 src/internal/git/git_test.go create mode 100644 src/internal/git/repository.go create mode 100644 src/internal/git/repository_test.go delete mode 100644 src/internal/packager/git/checkout.go delete mode 100644 src/internal/packager/git/clone.go delete mode 100644 src/internal/packager/git/common.go delete mode 100644 src/internal/packager/git/pull.go delete mode 100644 src/internal/packager/git/push.go diff --git a/go.mod b/go.mod index 6c345a7fe3..cc7d4d1417 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/distribution/reference v0.5.0 github.com/fairwindsops/pluto/v5 v5.18.4 github.com/fatih/color v1.17.0 + github.com/fluxcd/gitkit v0.6.0 github.com/fluxcd/helm-controller/api v1.0.1 github.com/fluxcd/pkg/apis/meta v1.5.0 github.com/fluxcd/source-controller/api v1.3.0 @@ -65,6 +66,7 @@ require ( require ( github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/gofrs/uuid v4.2.0+incompatible // indirect ) require ( @@ -244,7 +246,7 @@ require ( github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-billy/v5 v5.5.0 github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect diff --git a/go.sum b/go.sum index 0e511de64e..c66852dca2 100644 --- a/go.sum +++ b/go.sum @@ -706,6 +706,8 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg= +github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo= github.com/fluxcd/helm-controller/api v1.0.1 h1:Gn9qEVuif6D5+gHmVwTEZkR4+nmLOcOhKx4Sw2gL2EA= github.com/fluxcd/helm-controller/api v1.0.1/go.mod h1:/6AD5a2qjo/ttxVM8GR33syLZwqigta60DCLdy8GrME= github.com/fluxcd/pkg/apis/acl v0.3.0 h1:UOrKkBTOJK+OlZX7n8rWt2rdBmDCoTK+f5TY2LcZi8A= @@ -841,6 +843,8 @@ github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 951ae88ccf..31b71015cf 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -533,7 +533,7 @@ func findImagesforBBChartRepo(ctx context.Context, repo string, values chartutil spinner := message.NewProgressSpinner("Discovering images in %s", repo) defer spinner.Stop() - gitPath, err := helm.DownloadChartFromGitToTemp(ctx, repo, spinner) + gitPath, err := helm.DownloadChartFromGitToTemp(ctx, repo) if err != nil { return images, err } diff --git a/src/internal/git/fallback.go b/src/internal/git/fallback.go new file mode 100644 index 0000000000..960747413a --- /dev/null +++ b/src/internal/git/fallback.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package git contains functions for interacting with git repositories. +package git + +import ( + "context" + "io" + + "github.com/go-git/go-git/v5/plumbing" + + "github.com/zarf-dev/zarf/src/pkg/utils/exec" +) + +// gitCloneFallback is a fallback if go-git fails to clone a repo. +func (r *Repository) gitCloneFallback(ctx context.Context, gitURL string, ref plumbing.ReferenceName, shallow bool) error { + // If we can't clone with go-git, fallback to the host clone + // Only support "all tags" due to the azure clone url format including a username + cloneArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, r.path} + + // Don't clone all tags / refs if we're cloning a specific tag or branch. + if ref.IsTag() || ref.IsBranch() { + cloneArgs = append(cloneArgs, "--no-tags") + cloneArgs = append(cloneArgs, "-b", ref.Short()) + cloneArgs = append(cloneArgs, "--single-branch") + } + + // If this is a shallow clone set the depth to 1 + if shallow { + cloneArgs = append(cloneArgs, "--depth", "1") + } + + cloneExecConfig := exec.Config{ + Stdout: io.Discard, + Stderr: io.Discard, + } + _, _, err := exec.CmdWithContext(ctx, cloneExecConfig, "git", cloneArgs...) + if err != nil { + return err + } + + // If we're cloning the whole repo, we need to also fetch the other branches besides the default. + if ref == emptyRef { + fetchArgs := []string{"fetch", "--tags", "--update-head-ok", onlineRemoteName, "refs/*:refs/*"} + fetchExecConfig := exec.Config{ + Stdout: io.Discard, + Stderr: io.Discard, + Dir: r.path, + } + _, _, err := exec.CmdWithContext(ctx, fetchExecConfig, "git", fetchArgs...) + if err != nil { + return err + } + } + + return nil +} diff --git a/src/internal/git/git.go b/src/internal/git/git.go new file mode 100644 index 0000000000..506f2bea24 --- /dev/null +++ b/src/internal/git/git.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package git contains functions for interacting with git repositories. +package git + +import ( + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +const onlineRemoteName = "online-upstream" +const offlineRemoteName = "offline-downstream" +const emptyRef = "" + +// ParseRef parses the provided ref into a ReferenceName if it's not a hash. +func ParseRef(r string) plumbing.ReferenceName { + // If not a full ref, assume it's a tag at this point. + if !plumbing.IsHash(r) && !strings.HasPrefix(r, "refs/") { + r = fmt.Sprintf("refs/tags/%s", r) + } + // Set the reference name to the provided ref. + return plumbing.ReferenceName(r) +} diff --git a/src/internal/git/git_test.go b/src/internal/git/git_test.go new file mode 100644 index 0000000000..640d8a8b43 --- /dev/null +++ b/src/internal/git/git_test.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package git + +import ( + "testing" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/require" +) + +func TestParseRef(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + refPlain string + expectedRef plumbing.ReferenceName + }{ + { + name: "basic", + refPlain: "v1.0.0", + expectedRef: plumbing.ReferenceName("refs/tags/v1.0.0"), + }, + { + name: "basic", + refPlain: "refs/heads/branchname", + expectedRef: plumbing.ReferenceName("refs/heads/branchname"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ref := ParseRef(tt.refPlain) + require.Equal(t, tt.expectedRef, ref) + }) + } +} diff --git a/src/internal/git/repository.go b/src/internal/git/repository.go new file mode 100644 index 0000000000..5c920cb80a --- /dev/null +++ b/src/internal/git/repository.go @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package git + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + + "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/transform" + "github.com/zarf-dev/zarf/src/pkg/utils" +) + +// Open opens an existing local repository at the given path. +func Open(rootPath, address string) (*Repository, error) { + repoFolder, err := transform.GitURLtoFolderName(address) + if err != nil { + return nil, fmt.Errorf("unable to parse git url %s: %w", address, err) + } + repoPath := filepath.Join(rootPath, repoFolder) + + // Check that this package is using the new repo format (if not fallback to the format from <= 0.24.x) + _, err = os.Stat(repoPath) + if os.IsNotExist(err) { + repoFolder, err = transform.GitURLtoRepoName(address) + if err != nil { + return nil, fmt.Errorf("unable to parse git url %s: %w", address, err) + } + repoPath = filepath.Join(rootPath, repoFolder) + } + + return &Repository{ + path: repoPath, + }, nil +} + +// Clone clones a git repository to the given local path. +func Clone(ctx context.Context, rootPath, address string, shallow bool) (*Repository, error) { + // Split the remote url and the zarf reference + gitURLNoRef, refPlain, err := transform.GitURLSplitRef(address) + if err != nil { + return nil, err + } + + // Parse the ref from the git URL. + var ref plumbing.ReferenceName + if refPlain != emptyRef { + ref = ParseRef(refPlain) + } + + // Construct a path unique to this git repo + repoFolder, err := transform.GitURLtoFolderName(address) + if err != nil { + return nil, err + } + + r := &Repository{ + path: filepath.Join(rootPath, repoFolder), + } + + // Clone the repository + cloneOpts := &git.CloneOptions{ + URL: gitURLNoRef, + RemoteName: onlineRemoteName, + } + if ref.IsTag() || ref.IsBranch() { + cloneOpts.Tags = git.NoTags + cloneOpts.ReferenceName = ref + cloneOpts.SingleBranch = true + } + if shallow { + cloneOpts.Depth = 1 + } + gitCred, err := utils.FindAuthForHost(gitURLNoRef) + if err != nil { + return nil, err + } + if gitCred != nil { + cloneOpts.Auth = &gitCred.Auth + } + repo, err := git.PlainCloneContext(ctx, r.path, false, cloneOpts) + if err != nil { + message.Notef("Falling back to host 'git', failed to clone the repo %q with Zarf: %s", gitURLNoRef, err.Error()) + err := r.gitCloneFallback(ctx, gitURLNoRef, ref, shallow) + if err != nil { + return nil, err + } + } + + // If we're cloning the whole repo, we need to also fetch the other branches besides the default. + if ref == emptyRef { + fetchOpts := &git.FetchOptions{ + RemoteName: onlineRemoteName, + RefSpecs: []config.RefSpec{"refs/*:refs/*"}, + Tags: git.AllTags, + } + if gitCred != nil { + fetchOpts.Auth = &gitCred.Auth + } + if err := repo.FetchContext(ctx, fetchOpts); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return nil, err + } + } + + // Optionally checkout ref + if ref != emptyRef && !ref.IsBranch() { + // Remove the "refs/tags/" prefix from the ref. + stripped := strings.TrimPrefix(refPlain, "refs/tags/") + // Use the plain ref as part of the branch name so it is unique and doesn't conflict with other refs. + alias := fmt.Sprintf("zarf-ref-%s", stripped) + trunkBranchName := plumbing.NewBranchReferenceName(alias) + // Checkout the ref as a branch. + err := r.checkoutRefAsBranch(stripped, trunkBranchName) + if err != nil { + return nil, err + } + } + + return r, nil +} + +// Repository manages a local git repository. +type Repository struct { + path string +} + +// Path returns the local path the repository is stored at. +func (r *Repository) Path() string { + return r.path +} + +// Push pushes the repository to the remote git server. +func (r *Repository) Push(ctx context.Context, address, username, password string) error { + repo, err := git.PlainOpen(r.path) + if err != nil { + return fmt.Errorf("not a valid git repo or unable to open: %w", err) + } + + // Configure new remote + remote, err := repo.Remote(onlineRemoteName) + if err != nil { + return fmt.Errorf("unable to find the git remote: %w", err) + } + if len(remote.Config().URLs) == 0 { + return fmt.Errorf("repository has zero remotes configured") + } + targetURL, err := transform.GitURL(address, remote.Config().URLs[0], username) + if err != nil { + return fmt.Errorf("unable to transform the git url: %w", err) + } + // Remove any preexisting offlineRemotes (happens when a retry is triggered) + err = repo.DeleteRemote(offlineRemoteName) + if err != nil && !errors.Is(err, git.ErrRemoteNotFound) { + return err + } + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: offlineRemoteName, + URLs: []string{targetURL.String()}, + }) + if err != nil { + return fmt.Errorf("failed to create offline remote: %w", err) + } + + // Push to new remote + gitCred := http.BasicAuth{ + Username: username, + Password: password, + } + + // Fetch remote offline refs in case of old update or if multiple refs are specified in one package + // Attempt the fetch, if it fails, log a warning and continue trying to push (might as well try..) + fetchOptions := &git.FetchOptions{ + RemoteName: offlineRemoteName, + Auth: &gitCred, + RefSpecs: []config.RefSpec{ + "refs/heads/*:refs/heads/*", + "refs/tags/*:refs/tags/*", + }, + } + err = repo.FetchContext(ctx, fetchOptions) + if errors.Is(err, transport.ErrRepositoryNotFound) { + message.Debugf("Repo not yet available offline, skipping fetch...") + } else if errors.Is(err, git.ErrForceNeeded) { + message.Debugf("Repo fetch requires force, skipping fetch...") + } else if errors.Is(err, git.NoErrAlreadyUpToDate) { + message.Debugf("Repo already up-to-date, skipping fetch...") + } else if err != nil { + return fmt.Errorf("unable to fetch the git repo prior to push: %w", err) + } + + // Push all heads and tags to the offline remote + err = repo.PushContext(ctx, &git.PushOptions{ + RemoteName: offlineRemoteName, + Auth: &gitCred, + // TODO: (@JEFFMCCOY) add the parsing for the `+` force prefix (see https://github.com/zarf-dev/zarf/issues/1410) + //Force: isForce, + // If a provided refspec doesn't push anything, it is just ignored + RefSpecs: []config.RefSpec{ + "refs/heads/*:refs/heads/*", + "refs/tags/*:refs/tags/*", + }, + }) + if errors.Is(err, git.NoErrAlreadyUpToDate) { + message.Debug("Repo already up-to-date") + } else if errors.Is(err, plumbing.ErrObjectNotFound) { + return fmt.Errorf("unable to push repo due to likely shallow clone: %s", err.Error()) + } else if err != nil { + return fmt.Errorf("unable to push repo to the gitops service: %s", err.Error()) + } + + return nil +} +func (r *Repository) checkoutRefAsBranch(ref string, branch plumbing.ReferenceName) error { + repo, err := git.PlainOpen(r.path) + if err != nil { + return fmt.Errorf("not a valid git repo or unable to open: %w", err) + } + + var hash plumbing.Hash + if plumbing.IsHash(ref) { + hash = plumbing.NewHash(ref) + } else { + tagRef, err := repo.Tag(ref) + if err != nil { + return fmt.Errorf("failed to locate tag %s in repository: %w", ref, err) + } + hash = tagRef.Hash() + } + + objRef, err := repo.Object(plumbing.AnyObject, hash) + if err != nil { + return fmt.Errorf("an error occurred when getting the repo's object reference: %w", err) + } + + var commitHash plumbing.Hash + switch objRef := objRef.(type) { + case *object.Tag: + commitHash = objRef.Target + case *object.Commit: + commitHash = objRef.Hash + default: + return fmt.Errorf("hash type %s not supported", objRef.Type().String()) + } + + checkoutOpts := &git.CheckoutOptions{ + Hash: commitHash, + Branch: branch, + Create: true, + Force: true, + } + tree, err := repo.Worktree() + if err != nil { + return fmt.Errorf("unable to load the git repo: %w", err) + } + return tree.Checkout(checkoutOpts) +} diff --git a/src/internal/git/repository_test.go b/src/internal/git/repository_test.go new file mode 100644 index 0000000000..7e75319eee --- /dev/null +++ b/src/internal/git/repository_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package git + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/fluxcd/gitkit" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/stretchr/testify/require" + + "github.com/defenseunicorns/pkg/helpers/v2" + + "github.com/zarf-dev/zarf/src/test/testutil" +) + +func TestRepository(t *testing.T) { + t.Parallel() + ctx := testutil.TestContext(t) + + cfg := gitkit.Config{ + Dir: t.TempDir(), + AutoCreate: true, + } + gitSrv := gitkit.New(cfg) + err := gitSrv.Setup() + require.NoError(t, err) + srv := httptest.NewServer(http.HandlerFunc(gitSrv.ServeHTTP)) + t.Cleanup(func() { + srv.Close() + }) + + rootPath := t.TempDir() + repoName := "test" + repoAddress := fmt.Sprintf("%s/%s.git", srv.URL, repoName) + checksum := helpers.GetCRCHash(repoAddress) + expectedPath := fmt.Sprintf("%s-%d", repoName, checksum) + + storer := memory.NewStorage() + fs := memfs.New() + initRepo, err := git.Init(storer, fs) + require.NoError(t, err) + w, err := initRepo.Worktree() + require.NoError(t, err) + filePath := "test.txt" + newFile, err := fs.Create(filePath) + require.NoError(t, err) + _, err = newFile.Write([]byte("Hello World")) + require.NoError(t, err) + newFile.Close() + _, err = w.Add(filePath) + require.NoError(t, err) + _, err = w.Commit("Initial commit", &git.CommitOptions{ + Author: &object.Signature{ + Email: "example@example.com", + }, + }) + require.NoError(t, err) + _, err = initRepo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{repoAddress}, + }) + require.NoError(t, err) + err = initRepo.Push(&git.PushOptions{ + RemoteName: "origin", + }) + require.NoError(t, err) + + repo, err := Clone(ctx, rootPath, repoAddress, false) + require.NoError(t, err) + require.Equal(t, filepath.Join(rootPath, expectedPath), repo.Path()) + + repo, err = Open(rootPath, repoAddress) + require.NoError(t, err) + require.Equal(t, filepath.Join(rootPath, expectedPath), repo.Path()) +} diff --git a/src/internal/packager/git/checkout.go b/src/internal/packager/git/checkout.go deleted file mode 100644 index 5a9a960508..0000000000 --- a/src/internal/packager/git/checkout.go +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "fmt" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// CheckoutTag performs a `git checkout` of the provided tag to a detached HEAD. -func (g *Git) CheckoutTag(tag string) error { - options := &git.CheckoutOptions{ - Branch: ParseRef(tag), - } - return g.checkout(options) -} - -func (g *Git) checkoutRefAsBranch(ref string, branch plumbing.ReferenceName) error { - if plumbing.IsHash(ref) { - return g.checkoutHashAsBranch(plumbing.NewHash(ref), branch) - } - - return g.checkoutTagAsBranch(ref, branch) -} - -// checkoutTagAsBranch performs a `git checkout` of the provided tag but rather -// than checking out to a detached head, checks out to the provided branch ref -// It will delete the branch provided if it exists. -func (g *Git) checkoutTagAsBranch(tag string, branch plumbing.ReferenceName) error { - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - tagRef, err := repo.Tag(tag) - if err != nil { - return fmt.Errorf("failed to locate tag (%s) in repository: %w", tag, err) - } - - return g.checkoutHashAsBranch(tagRef.Hash(), branch) -} - -// checkoutHashAsBranch performs a `git checkout` of the commit hash associated -// with the provided hash -// It will delete the branch provided if it exists. -func (g *Git) checkoutHashAsBranch(hash plumbing.Hash, branch plumbing.ReferenceName) error { - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - objRef, err := repo.Object(plumbing.AnyObject, hash) - if err != nil { - return fmt.Errorf("an error occurred when getting the repo's object reference: %w", err) - } - - var commitHash plumbing.Hash - switch objRef := objRef.(type) { - case *object.Tag: - commitHash = objRef.Target - case *object.Commit: - commitHash = objRef.Hash - default: - return fmt.Errorf("hash type %s not supported", objRef.Type().String()) - } - - options := &git.CheckoutOptions{ - Hash: commitHash, - Branch: branch, - Create: true, - Force: true, - } - return g.checkout(options) -} - -// checkout performs a `git checkout` on the path provided using the options provided -// It assumes the caller knows what to do and does not perform any safety checks. -func (g *Git) checkout(checkoutOptions *git.CheckoutOptions) error { - // Open the given repo - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - // Get the working tree so we can change refs - tree, err := repo.Worktree() - if err != nil { - return fmt.Errorf("unable to load the git repo: %w", err) - } - - // Perform the checkout - return tree.Checkout(checkoutOptions) -} diff --git a/src/internal/packager/git/clone.go b/src/internal/packager/git/clone.go deleted file mode 100644 index 2f607670e7..0000000000 --- a/src/internal/packager/git/clone.go +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "context" - "errors" - "strings" - - "github.com/go-git/go-git/v5" - goConfig "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing" - "github.com/zarf-dev/zarf/src/pkg/message" - "github.com/zarf-dev/zarf/src/pkg/utils" - "github.com/zarf-dev/zarf/src/pkg/utils/exec" -) - -// clone performs a `git clone` of a given repo. -func (g *Git) clone(ctx context.Context, gitURL string, ref plumbing.ReferenceName, shallow bool) error { - cloneOptions := &git.CloneOptions{ - URL: gitURL, - Progress: g.Spinner, - RemoteName: onlineRemoteName, - } - - // Don't clone all tags / refs if we're cloning a specific tag or branch. - if ref.IsTag() || ref.IsBranch() { - cloneOptions.Tags = git.NoTags - cloneOptions.ReferenceName = ref - cloneOptions.SingleBranch = true - } - - // If this is a shallow clone set the depth to 1 - if shallow { - cloneOptions.Depth = 1 - } - - // Setup git credentials if we have them, ignore if we don't. - gitCred, err := utils.FindAuthForHost(gitURL) - if err != nil { - return err - } - if gitCred != nil { - cloneOptions.Auth = &gitCred.Auth - } - - // Clone the given repo. - repo, err := git.PlainClone(g.GitPath, false, cloneOptions) - if err != nil { - message.Notef("Falling back to host 'git', failed to clone the repo %q with Zarf: %s", gitURL, err.Error()) - return g.gitCloneFallback(ctx, gitURL, ref, shallow) - } - - // If we're cloning the whole repo, we need to also fetch the other branches besides the default. - if ref == emptyRef { - fetchOpts := &git.FetchOptions{ - RemoteName: onlineRemoteName, - Progress: g.Spinner, - RefSpecs: []goConfig.RefSpec{"refs/*:refs/*"}, - Tags: git.AllTags, - } - - if gitCred != nil { - fetchOpts.Auth = &gitCred.Auth - } - - if err := repo.Fetch(fetchOpts); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { - return err - } - } - - return nil -} - -// gitCloneFallback is a fallback if go-git fails to clone a repo. -func (g *Git) gitCloneFallback(ctx context.Context, gitURL string, ref plumbing.ReferenceName, shallow bool) error { - // If we can't clone with go-git, fallback to the host clone - // Only support "all tags" due to the azure clone url format including a username - cloneArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, g.GitPath} - - // Don't clone all tags / refs if we're cloning a specific tag or branch. - if ref.IsTag() || ref.IsBranch() { - cloneArgs = append(cloneArgs, "--no-tags") - cloneArgs = append(cloneArgs, "-b", ref.Short()) - cloneArgs = append(cloneArgs, "--single-branch") - } - - // If this is a shallow clone set the depth to 1 - if shallow { - cloneArgs = append(cloneArgs, "--depth", "1") - } - - cloneExecConfig := exec.Config{ - Stdout: g.Spinner, - Stderr: g.Spinner, - } - - message.Command("git %s", strings.Join(cloneArgs, " ")) - - _, _, err := exec.CmdWithContext(ctx, cloneExecConfig, "git", cloneArgs...) - if err != nil { - return err - } - - // If we're cloning the whole repo, we need to also fetch the other branches besides the default. - if ref == emptyRef { - fetchArgs := []string{"fetch", "--tags", "--update-head-ok", onlineRemoteName, "refs/*:refs/*"} - - fetchExecConfig := exec.Config{ - Stdout: g.Spinner, - Stderr: g.Spinner, - Dir: g.GitPath, - } - - message.Command("git %s", strings.Join(fetchArgs, " ")) - - _, _, err := exec.CmdWithContext(ctx, fetchExecConfig, "git", fetchArgs...) - if err != nil { - return err - } - } - - return nil -} diff --git a/src/internal/packager/git/common.go b/src/internal/packager/git/common.go deleted file mode 100644 index 6864d3dea5..0000000000 --- a/src/internal/packager/git/common.go +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "fmt" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/zarf-dev/zarf/src/pkg/message" - "github.com/zarf-dev/zarf/src/types" -) - -// Git is the main struct for managing git repositories. -type Git struct { - // Server is the git server configuration. - Server types.GitServerInfo - // Spinner is an optional spinner to use for long running operations. - Spinner *message.Spinner - // Target working directory for the git repository. - GitPath string -} - -const onlineRemoteName = "online-upstream" -const offlineRemoteName = "offline-downstream" -const emptyRef = "" - -// New creates a new git instance with the provided server config. -func New(server types.GitServerInfo) *Git { - return &Git{ - Server: server, - } -} - -// NewWithSpinner creates a new git instance with the provided server config and spinner. -func NewWithSpinner(server types.GitServerInfo, spinner *message.Spinner) *Git { - return &Git{ - Server: server, - Spinner: spinner, - } -} - -// ParseRef parses the provided ref into a ReferenceName if it's not a hash. -func ParseRef(r string) plumbing.ReferenceName { - // If not a full ref, assume it's a tag at this point. - if !plumbing.IsHash(r) && !strings.HasPrefix(r, "refs/") { - r = fmt.Sprintf("refs/tags/%s", r) - } - - // Set the reference name to the provided ref. - return plumbing.ReferenceName(r) -} diff --git a/src/internal/packager/git/pull.go b/src/internal/packager/git/pull.go deleted file mode 100644 index 5d9e6f1006..0000000000 --- a/src/internal/packager/git/pull.go +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "context" - "fmt" - "path" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/zarf-dev/zarf/src/config" - "github.com/zarf-dev/zarf/src/pkg/transform" - "github.com/zarf-dev/zarf/src/pkg/utils" -) - -// DownloadRepoToTemp clones or updates a repo into a temp folder to perform ephemeral actions (i.e. process chart repos). -func (g *Git) DownloadRepoToTemp(ctx context.Context, gitURL string) error { - g.Spinner.Updatef("g.DownloadRepoToTemp(%s)", gitURL) - - path, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) - if err != nil { - return fmt.Errorf("unable to create tmpdir: %w", err) - } - - // If downloading to temp, set this as a shallow clone to only pull the exact - // gitURL w/ ref that was specified since we will throw away git history anyway - if err = g.Pull(ctx, gitURL, path, true); err != nil { - return fmt.Errorf("unable to pull the git repo at %s: %w", gitURL, err) - } - - return nil -} - -// Pull clones or updates a git repository into the target folder. -func (g *Git) Pull(ctx context.Context, gitURL, targetFolder string, shallow bool) error { - g.Spinner.Updatef("Processing git repo %s", gitURL) - - // Split the remote url and the zarf reference - gitURLNoRef, refPlain, err := transform.GitURLSplitRef(gitURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - - // Parse the ref from the git URL. - if refPlain != emptyRef { - ref = ParseRef(refPlain) - } - - // Construct a path unique to this git repo - repoFolder, err := transform.GitURLtoFolderName(gitURL) - if err != nil { - return err - } - - g.GitPath = path.Join(targetFolder, repoFolder) - - // Clone the git repository. - err = g.clone(ctx, gitURLNoRef, ref, shallow) - if err != nil { - return fmt.Errorf("not a valid git repo or unable to clone (%s): %w", gitURL, err) - } - - if ref != emptyRef && !ref.IsBranch() { - // Remove the "refs/tags/" prefix from the ref. - stripped := strings.TrimPrefix(refPlain, "refs/tags/") - - // Use the plain ref as part of the branch name so it is unique and doesn't conflict with other refs. - alias := fmt.Sprintf("zarf-ref-%s", stripped) - trunkBranchName := plumbing.NewBranchReferenceName(alias) - - // Checkout the ref as a branch. - return g.checkoutRefAsBranch(stripped, trunkBranchName) - } - - return nil -} diff --git a/src/internal/packager/git/push.go b/src/internal/packager/git/push.go deleted file mode 100644 index 60b7b2bdf6..0000000000 --- a/src/internal/packager/git/push.go +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package git contains functions for interacting with git repositories. -package git - -import ( - "errors" - "fmt" - "os" - "path" - - "github.com/go-git/go-git/v5" - goConfig "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/zarf-dev/zarf/src/pkg/message" - "github.com/zarf-dev/zarf/src/pkg/transform" -) - -// PushRepo pushes a git repository from the local path to the configured git server. -func (g *Git) PushRepo(srcURL, targetFolder string) error { - spinner := message.NewProgressSpinner("Processing git repo %s", srcURL) - defer spinner.Stop() - - // Setup git paths, including a unique name for the repo based on the hash of the git URL to avoid conflicts. - repoFolder, err := transform.GitURLtoFolderName(srcURL) - if err != nil { - return fmt.Errorf("unable to parse git url (%s): %w", srcURL, err) - } - repoPath := path.Join(targetFolder, repoFolder) - - // Check that this package is using the new repo format (if not fallback to the format from <= 0.24.x) - _, err = os.Stat(repoPath) - if os.IsNotExist(err) { - repoFolder, err = transform.GitURLtoRepoName(srcURL) - if err != nil { - return fmt.Errorf("unable to parse git url (%s): %w", srcURL, err) - } - repoPath = path.Join(targetFolder, repoFolder) - } - - g.GitPath = repoPath - - repo, err := g.prepRepoForPush() - if err != nil { - return fmt.Errorf("could not prepare the repo for push: %w", err) - } - - if err := g.push(repo, spinner); err != nil { - return fmt.Errorf("failed to push the git repo %q: %w", repoFolder, err) - } - - spinner.Success() - return nil -} - -func (g *Git) prepRepoForPush() (*git.Repository, error) { - // Open the given repo - repo, err := git.PlainOpen(g.GitPath) - if err != nil { - return nil, fmt.Errorf("not a valid git repo or unable to open: %w", err) - } - - // Get the upstream URL - remote, err := repo.Remote(onlineRemoteName) - if err != nil { - return nil, fmt.Errorf("unable to find the git remote: %w", err) - } - - remoteURL := remote.Config().URLs[0] - targetURL, err := transform.GitURL(g.Server.Address, remoteURL, g.Server.PushUsername) - if err != nil { - return nil, fmt.Errorf("unable to transform the git url: %w", err) - } - message.Debugf("Rewrite git URL: %s -> %s", remoteURL, targetURL.String()) - // Remove any preexisting offlineRemotes (happens when a retry is triggered) - _ = repo.DeleteRemote(offlineRemoteName) - - _, err = repo.CreateRemote(&goConfig.RemoteConfig{ - Name: offlineRemoteName, - URLs: []string{targetURL.String()}, - }) - if err != nil { - return nil, fmt.Errorf("failed to create offline remote: %w", err) - } - - return repo, nil -} - -func (g *Git) push(repo *git.Repository, spinner *message.Spinner) error { - gitCred := http.BasicAuth{ - Username: g.Server.PushUsername, - Password: g.Server.PushPassword, - } - - // Fetch remote offline refs in case of old update or if multiple refs are specified in one package - fetchOptions := &git.FetchOptions{ - RemoteName: offlineRemoteName, - Auth: &gitCred, - RefSpecs: []goConfig.RefSpec{ - "refs/heads/*:refs/heads/*", - "refs/tags/*:refs/tags/*", - }, - } - - // Attempt the fetch, if it fails, log a warning and continue trying to push (might as well try..) - err := repo.Fetch(fetchOptions) - if errors.Is(err, transport.ErrRepositoryNotFound) { - message.Debugf("Repo not yet available offline, skipping fetch...") - } else if errors.Is(err, git.ErrForceNeeded) { - message.Debugf("Repo fetch requires force, skipping fetch...") - } else if errors.Is(err, git.NoErrAlreadyUpToDate) { - message.Debugf("Repo already up-to-date, skipping fetch...") - } else if err != nil { - return fmt.Errorf("unable to fetch the git repo prior to push: %w", err) - } - - // Push all heads and tags to the offline remote - err = repo.Push(&git.PushOptions{ - RemoteName: offlineRemoteName, - Auth: &gitCred, - Progress: spinner, - // TODO: (@JEFFMCCOY) add the parsing for the `+` force prefix (see https://github.com/zarf-dev/zarf/issues/1410) - //Force: isForce, - // If a provided refspec doesn't push anything, it is just ignored - RefSpecs: []goConfig.RefSpec{ - "refs/heads/*:refs/heads/*", - "refs/tags/*:refs/tags/*", - }, - }) - - if errors.Is(err, git.NoErrAlreadyUpToDate) { - message.Debug("Repo already up-to-date") - } else if errors.Is(err, plumbing.ErrObjectNotFound) { - return fmt.Errorf("unable to push repo due to likely shallow clone: %s", err.Error()) - } else if err != nil { - return fmt.Errorf("unable to push repo to the gitops service: %s", err.Error()) - } - - return nil -} diff --git a/src/internal/packager/helm/repo.go b/src/internal/packager/helm/repo.go index dad0e59aab..24f3a7f4b0 100644 --- a/src/internal/packager/helm/repo.go +++ b/src/internal/packager/helm/repo.go @@ -15,11 +15,10 @@ import ( "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/config/lang" - "github.com/zarf-dev/zarf/src/internal/packager/git" + "github.com/zarf-dev/zarf/src/internal/git" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/transform" "github.com/zarf-dev/zarf/src/pkg/utils" - "github.com/zarf-dev/zarf/src/types" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/cli" @@ -119,7 +118,7 @@ func (h *Helm) PackageChartFromGit(ctx context.Context, cosignKeyPath string) er defer spinner.Stop() // Retrieve the repo containing the chart - gitPath, err := DownloadChartFromGitToTemp(ctx, h.chart.URL, spinner) + gitPath, err := DownloadChartFromGitToTemp(ctx, h.chart.URL) if err != nil { return err } @@ -233,17 +232,16 @@ func (h *Helm) DownloadPublishedChart(ctx context.Context, cosignKeyPath string) } // DownloadChartFromGitToTemp downloads a chart from git into a temp directory -func DownloadChartFromGitToTemp(ctx context.Context, url string, spinner *message.Spinner) (string, error) { - // Create the Git configuration and download the repo - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - - // Download the git repo to a temporary directory - err := gitCfg.DownloadRepoToTemp(ctx, url) +func DownloadChartFromGitToTemp(ctx context.Context, url string) (string, error) { + path, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - return "", fmt.Errorf("unable to download the git repo %s: %w", url, err) + return "", fmt.Errorf("unable to create tmpdir: %w", err) } - - return gitCfg.GitPath, nil + repository, err := git.Clone(ctx, path, url, true) + if err != nil { + return "", err + } + return repository.Path(), nil } func (h *Helm) finalizeChartPackage(ctx context.Context, saved, cosignKeyPath string) error { diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 99b0f572b4..2b32ac1ff1 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -21,7 +21,7 @@ import ( "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/config/lang" "github.com/zarf-dev/zarf/src/extensions/bigbang" - "github.com/zarf-dev/zarf/src/internal/packager/git" + "github.com/zarf-dev/zarf/src/internal/git" "github.com/zarf-dev/zarf/src/internal/packager/helm" "github.com/zarf-dev/zarf/src/internal/packager/images" "github.com/zarf-dev/zarf/src/internal/packager/kustomize" @@ -513,8 +513,8 @@ func (pc *PackageCreator) addComponent(ctx context.Context, component types.Zarf for _, url := range component.Repos { // Pull all the references if there is no `@` in the string. - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - if err := gitCfg.Pull(ctx, url, componentPaths.Repos, false); err != nil { + _, err := git.Clone(ctx, componentPaths.Repos, url, false) + if err != nil { return fmt.Errorf("unable to pull git repo %s: %w", url, err) } } diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 428f2e1726..0bc5fcfedb 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -25,8 +25,8 @@ import ( "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/internal/git" "github.com/zarf-dev/zarf/src/internal/gitea" - "github.com/zarf-dev/zarf/src/internal/packager/git" "github.com/zarf-dev/zarf/src/internal/packager/helm" "github.com/zarf-dev/zarf/src/internal/packager/images" "github.com/zarf-dev/zarf/src/internal/packager/template" @@ -545,6 +545,11 @@ func (p *Packager) pushImagesToRegistry(ctx context.Context, componentImages []s // Push all of the components git repos to the configured git server. func (p *Packager) pushReposToRepository(ctx context.Context, reposPath string, repos []string) error { for _, repoURL := range repos { + repository, err := git.Open(reposPath, repoURL) + if err != nil { + return err + } + // Create an anonymous function to push the repo to the Zarf git server tryPush := func() error { namespace, name, port, err := serviceInfoFromServiceURL(p.state.GitServer.Address) @@ -560,7 +565,6 @@ func (p *Packager) pushReposToRepository(ctx context.Context, reposPath string, return err } } - tunnel, err := p.cluster.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) if err != nil { return err @@ -570,18 +574,15 @@ func (p *Packager) pushReposToRepository(ctx context.Context, reposPath string, return err } defer tunnel.Close() - gitClient := git.New(p.state.GitServer) - gitClient.Server.Address = tunnel.HTTPEndpoint() giteaClient, err := gitea.NewClient(tunnel.HTTPEndpoint(), p.state.GitServer.PushUsername, p.state.GitServer.PushPassword) if err != nil { return err } return tunnel.Wrap(func() error { - err = gitClient.PushRepo(repoURL, reposPath) + err = repository.Push(ctx, tunnel.HTTPEndpoint(), p.state.GitServer.PushUsername, p.state.GitServer.PushPassword) if err != nil { return err } - // Add the read-only user to this repo repoName, err := transform.GitURLtoRepoName(repoURL) if err != nil { @@ -595,8 +596,11 @@ func (p *Packager) pushReposToRepository(ctx context.Context, reposPath string, }) } - gitClient := git.New(p.state.GitServer) - return gitClient.PushRepo(repoURL, reposPath) + err = repository.Push(ctx, p.state.GitServer.Address, p.state.GitServer.PushUsername, p.state.GitServer.PushPassword) + if err != nil { + return err + } + return nil } // Try repo push up to retry limit diff --git a/src/pkg/packager/filters/diff.go b/src/pkg/packager/filters/diff.go index bbba9ab789..7b65f6f576 100644 --- a/src/pkg/packager/filters/diff.go +++ b/src/pkg/packager/filters/diff.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/go-git/go-git/v5/plumbing" - "github.com/zarf-dev/zarf/src/internal/packager/git" + "github.com/zarf-dev/zarf/src/internal/git" "github.com/zarf-dev/zarf/src/pkg/transform" "github.com/zarf-dev/zarf/src/types" )