Skip to content

Commit

Permalink
update klone destination in-place using rsync
Browse files Browse the repository at this point in the history
Signed-off-by: Tim Ramlot <42113979+inteon@users.noreply.github.com>
  • Loading branch information
inteon committed May 3, 2024
1 parent 96998e1 commit 46e85aa
Show file tree
Hide file tree
Showing 6 changed files with 144 additions and 106 deletions.
50 changes: 2 additions & 48 deletions cmd/sync.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
package cmd

import (
"fmt"
"os"
"path/filepath"

"github.com/cert-manager/klone/pkg/cache"
"github.com/cert-manager/klone/pkg/download/git"
"github.com/cert-manager/klone/pkg/mod"

"github.com/cert-manager/klone/pkg/sync"
"github.com/spf13/cobra"
)

Expand All @@ -23,48 +18,7 @@ func NewSyncCommand() *cobra.Command {
return err
}

wrkDir := mod.WorkDir(workDirPath)
if err := wrkDir.FetchTargets(
func(_ string, _ string, src *mod.KloneSource) error {
src.RepoPath = filepath.Join(".", filepath.Clean(filepath.Join("/", src.RepoPath)))

if src.RepoHash == "" {
hash, err := git.GetHash(src.RepoURL, src.RepoRef)
if err != nil {
return err
}

src.RepoHash = hash
}

return nil
},
func(target string, srcs mod.KloneFolder) error {
if err := os.RemoveAll(filepath.Join(workDirPath, target)); err != nil {
return err
}

if err := os.MkdirAll(filepath.Join(workDirPath, target), 0755); err != nil {
return err
}

for _, src := range srcs {
if err := cache.CloneWithCache(filepath.Join(workDirPath, target, src.FolderName), src.KloneSource, git.Get); err != nil {
return err
}
}

return nil
},
); err != nil {
return fmt.Errorf("failed to fetch targets: %w", err)
}

if err := cache.CleanupOldCacheItems(); err != nil {
return fmt.Errorf("failed to cleanup old cache items: %w", err)
}

return nil
return sync.SyncFolder(workDirPath, false)
},
}

Expand Down
48 changes: 2 additions & 46 deletions cmd/upgrade.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
package cmd

import (
"fmt"
"os"
"path/filepath"

"github.com/cert-manager/klone/pkg/cache"
"github.com/cert-manager/klone/pkg/download/git"
"github.com/cert-manager/klone/pkg/mod"

"github.com/cert-manager/klone/pkg/sync"
"github.com/spf13/cobra"
)

Expand All @@ -23,46 +18,7 @@ func NewUpgradeCommand() *cobra.Command {
return err
}

wrkDir := mod.WorkDir(workDirPath)
if err := wrkDir.FetchTargets(
func(_ string, _ string, src *mod.KloneSource) error {
src.RepoPath = filepath.Join(".", filepath.Clean(filepath.Join("/", src.RepoPath)))

hash, err := git.GetHash(src.RepoURL, src.RepoRef)
if err != nil {
return err
}

src.RepoHash = hash

return nil
},
func(target string, srcs mod.KloneFolder) error {
if err := os.RemoveAll(filepath.Join(workDirPath, target)); err != nil {
return err
}

if err := os.MkdirAll(filepath.Join(workDirPath, target), 0755); err != nil {
return err
}

for _, src := range srcs {
if err := cache.CloneWithCache(filepath.Join(workDirPath, target, src.FolderName), src.KloneSource, git.Get); err != nil {
return err
}
}

return nil
},
); err != nil {
return fmt.Errorf("failed to fetch targets: %w", err)
}

if err := cache.CleanupOldCacheItems(); err != nil {
return fmt.Errorf("failed to cleanup old cache items: %w", err)
}

return nil
return sync.SyncFolder(workDirPath, true)
},
}

Expand Down
2 changes: 0 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ module github.com/cert-manager/klone
go 1.21.1

require (
github.com/otiai10/copy v1.12.0
github.com/rogpeppe/go-internal v1.11.0
github.com/spf13/cobra v1.7.0
gopkg.in/yaml.v3 v3.0.1
Expand All @@ -12,5 +11,4 @@ require (
require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
)
6 changes: 0 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,19 +1,13 @@
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY=
github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
25 changes: 21 additions & 4 deletions pkg/cache/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ package cache
import (
"crypto/sha256"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"time"

"github.com/cert-manager/klone/pkg/mod"
cp "github.com/otiai10/copy"
)

func calculateCacheKey(src mod.KloneSource) string {
Expand Down Expand Up @@ -73,17 +74,33 @@ func CloneWithCache(
return err
}

if err := os.RemoveAll(destPath); err != nil {
if err := os.MkdirAll(destPath, 0755); err != nil {
return err
}

if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
if err := runRsyncCmd(cachePath, os.Stdout, os.Stderr, "-aEq", ".", destPath); err != nil {
return err
}

if err := cp.Copy(cachePath, destPath); err != nil {
return nil
}

func runRsyncCmd(root string, stdout io.Writer, stderr io.Writer, args ...string) error {
cmd := exec.Command("rsync", args...)

cmd.Dir = root
cmd.Env = append(os.Environ(), cmd.Env...)

cmd.Stdout = stdout
cmd.Stderr = stderr

if err := cmd.Start(); err != nil {
return err
}

if err := cmd.Wait(); err != nil {
return fmt.Errorf("rsync command failed: %v", err)
}

return nil
}
119 changes: 119 additions & 0 deletions pkg/sync/sync.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package sync

import (
"fmt"
"os"
"path/filepath"

"github.com/cert-manager/klone/pkg/cache"
"github.com/cert-manager/klone/pkg/download/git"
"github.com/cert-manager/klone/pkg/mod"
)

func SyncFolder(workDirPath string, forceUpgrade bool) error {
wrkDir := mod.WorkDir(workDirPath)
if err := wrkDir.FetchTargets(
func(_ string, _ string, src *mod.KloneSource) error {
src.RepoPath = filepath.Join(".", filepath.Clean(filepath.Join("/", src.RepoPath)))

if src.RepoHash == "" || forceUpgrade {
hash, err := git.GetHash(src.RepoURL, src.RepoRef)
if err != nil {
return err
}

src.RepoHash = hash
}

return nil
},
func(target string, srcs mod.KloneFolder) error {
folders := newTreeNode()
for _, src := range srcs {
folders.Add(filepath.SplitList(src.FolderName)...)
}

if err := os.MkdirAll(filepath.Join(workDirPath, target), 0755); err != nil {
return err
}

// 1) Remove all folders that are not defined in srcs
if err := folders.Cleanup(filepath.Join(workDirPath, target)); err != nil {
return err
}

// 2) Sync all folders with cached files
for _, src := range srcs {
if err := cache.CloneWithCache(filepath.Join(workDirPath, target, src.FolderName), src.KloneSource, git.Get); err != nil {
return err
}
}

return nil
},
); err != nil {
return fmt.Errorf("failed to fetch targets: %w", err)
}

if err := cache.CleanupOldCacheItems(); err != nil {
return fmt.Errorf("failed to cleanup old cache items: %w", err)
}

return nil

}

type treeNode struct {
isLeaf bool
children map[string]*treeNode
}

func newTreeNode() *treeNode {
return &treeNode{
isLeaf: false,
children: make(map[string]*treeNode),
}
}

func (tn *treeNode) Add(pathSegments ...string) {
if len(pathSegments) == 0 {
tn.isLeaf = true
return
}

if _, ok := tn.children[pathSegments[0]]; !ok {
tn.children[pathSegments[0]] = newTreeNode()
}

tn.children[pathSegments[0]].Add(pathSegments[1:]...)
}

func (tn treeNode) Cleanup(root string) error {
if tn.isLeaf {
return nil
}

entries, err := os.ReadDir(root)
if err != nil {
return err
}

for _, entry := range entries {
entryName := entry.Name()
if _, ok := tn.children[entryName]; ok {
continue
}

if err := os.RemoveAll(filepath.Join(root, entryName)); err != nil {
return err
}
}

for name, node := range tn.children {
if err := node.Cleanup(filepath.Join(root, name)); err != nil {
return err
}
}

return nil
}

0 comments on commit 46e85aa

Please sign in to comment.