Merge pull request #743 from fluxcd/refactor-artifact-fetcher

Refactor: Acquire artifacts with `fluxcd/pkg/http/fetch`

controllers/kustomization_controller.go

@@ -50,12 +50,14 @@ import (
 
 	apiacl "github.com/fluxcd/pkg/apis/acl"
 	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/http/fetch"
 	"github.com/fluxcd/pkg/runtime/acl"
 	runtimeClient "github.com/fluxcd/pkg/runtime/client"
 	"github.com/fluxcd/pkg/runtime/events"
 	"github.com/fluxcd/pkg/runtime/metrics"
 	"github.com/fluxcd/pkg/runtime/predicates"
 	"github.com/fluxcd/pkg/ssa"
+	"github.com/fluxcd/pkg/tar"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
@@ -74,7 +76,7 @@ import (
 // KustomizationReconciler reconciles a Kustomization object
 type KustomizationReconciler struct {
 	client.Client
-	artifactFetcher       *ArtifactFetcher
+	artifactFetcher       *fetch.ArchiveFetcher
 	requeueDependency     time.Duration
 	Scheme                *runtime.Scheme
 	EventRecorder         kuberecorder.EventRecorder
@@ -124,7 +126,7 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom
 
 	r.requeueDependency = opts.DependencyRequeueInterval
 	r.statusManager = fmt.Sprintf("gotk-%s", r.ControllerName)
-	r.artifactFetcher = NewArtifactFetcher(opts.HTTPRetry)
+	r.artifactFetcher = fetch.NewArchiveFetcher(opts.HTTPRetry, tar.UnlimitedUntarSize, os.Getenv("SOURCE_CONTROLLER_LOCALHOST"))
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&kustomizev1.Kustomization{}, builder.WithPredicates(
@@ -270,7 +272,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	reconciledKustomization, reconcileErr := r.reconcile(ctx, *kustomization.DeepCopy(), source)
 
 	// requeue if the artifact is not found
-	if reconcileErr == ArtifactNotFoundError {
+	if reconcileErr == fetch.FileNotFoundError {
 		msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.requeueDependency.String())
 		log.Info(msg)
 		if err := r.patchStatus(ctx, req, kustomizev1.KustomizationProgressing(kustomization, msg).Status); err != nil {
@@ -332,7 +334,7 @@ func (r *KustomizationReconciler) reconcile(
 	defer os.RemoveAll(tmpDir)
 
 	// download artifact and extract files
-	err = r.artifactFetcher.Fetch(source.GetArtifact(), tmpDir)
+	err = r.artifactFetcher.Fetch(source.GetArtifact().URL, source.GetArtifact().Checksum, tmpDir)
 	if err != nil {
 		return kustomizev1.KustomizationNotReady(
 			kustomization,

controllers/kustomization_decryptor_test.go

@@ -66,11 +66,11 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
 	artifactName := "sops-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/sops", artifactName)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/sops", artifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	overlayArtifactName := "sops-" + randStringRunes(5)
-	overlayChecksum, err := createArtifact(testServer, "testdata/test-dotenv", overlayArtifactName)
+	overlayChecksum, err := testServer.ArtifactFromDir("testdata/test-dotenv", overlayArtifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{

controllers/kustomization_transformer_test.go

@@ -50,7 +50,7 @@ func TestKustomizationReconciler_KustomizeTransformer(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	artifactFile := "patch-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/transformers", artifactFile)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/transformers", artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{
@@ -173,7 +173,7 @@ func TestKustomizationReconciler_KustomizeTransformerFiles(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	artifactFile := "patch-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/file-transformer", artifactFile)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/file-transformer", artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{
@@ -292,7 +292,7 @@ func TestKustomizationReconciler_FluxTransformers(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
 	artifactFile := "patch-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/patch", artifactFile)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/patch", artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{

controllers/kustomization_validation_test.go

@@ -43,11 +43,11 @@ func TestKustomizationReconciler_Validation(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
 	artifactName := "val-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/invalid/plain", artifactName)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/invalid/plain", artifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	overlayArtifactName := "val-" + randStringRunes(5)
-	overlayChecksum, err := createArtifact(testServer, "testdata/invalid/overlay", overlayArtifactName)
+	overlayChecksum, err := testServer.ArtifactFromDir("testdata/invalid/overlay", overlayArtifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{

controllers/suite_test.go

@@ -17,17 +17,12 @@ limitations under the License.
 package controllers
 
 import (
-	"archive/tar"
-	"compress/gzip"
 	"context"
-	"crypto/sha1"
 	"crypto/sha256"
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
@@ -288,95 +283,6 @@ func applyGitRepository(objKey client.ObjectKey, artifactName string, revision s
 	return nil
 }
 
-func createArtifact(artifactServer *testserver.ArtifactServer, fixture, path string) (string, error) {
-	if f, err := os.Stat(fixture); os.IsNotExist(err) || !f.IsDir() {
-		return "", fmt.Errorf("invalid fixture path: %s", fixture)
-	}
-	f, err := os.Create(filepath.Join(artifactServer.Root(), path))
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		if err != nil {
-			os.Remove(f.Name())
-		}
-	}()
-
-	h := sha1.New()
-
-	mw := io.MultiWriter(h, f)
-	gw := gzip.NewWriter(mw)
-	tw := tar.NewWriter(gw)
-
-	if err = filepath.Walk(fixture, func(p string, fi os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Ignore anything that is not a file (directories, symlinks)
-		if !fi.Mode().IsRegular() {
-			return nil
-		}
-
-		// Ignore dotfiles
-		if strings.HasPrefix(fi.Name(), ".") {
-			return nil
-		}
-
-		header, err := tar.FileInfoHeader(fi, p)
-		if err != nil {
-			return err
-		}
-		// The name needs to be modified to maintain directory structure
-		// as tar.FileInfoHeader only has access to the base name of the file.
-		// Ref: https://golang.org/src/archive/tar/common.go?#L626
-		relFilePath := p
-		if filepath.IsAbs(fixture) {
-			relFilePath, err = filepath.Rel(fixture, p)
-			if err != nil {
-				return err
-			}
-		}
-		header.Name = relFilePath
-
-		if err := tw.WriteHeader(header); err != nil {
-			return err
-		}
-
-		f, err := os.Open(p)
-		if err != nil {
-			f.Close()
-			return err
-		}
-		if _, err := io.Copy(tw, f); err != nil {
-			f.Close()
-			return err
-		}
-		return f.Close()
-	}); err != nil {
-		return "", err
-	}
-
-	if err := tw.Close(); err != nil {
-		gw.Close()
-		f.Close()
-		return "", err
-	}
-	if err := gw.Close(); err != nil {
-		f.Close()
-		return "", err
-	}
-	if err := f.Close(); err != nil {
-		return "", err
-	}
-
-	if err := os.Chmod(f.Name(), 0644); err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
 func createVaultTestInstance() (*dockertest.Pool, *dockertest.Resource, error) {
 	// uses a sensible default on windows (tcp/http) and linux/osx (socket)
 	pool, err := dockertest.NewPool("")

go.mod

@@ -24,15 +24,15 @@ require (
 	github.com/fluxcd/pkg/apis/acl v0.1.0
 	github.com/fluxcd/pkg/apis/kustomize v0.6.0
 	github.com/fluxcd/pkg/apis/meta v0.17.0
+	github.com/fluxcd/pkg/http/fetch v0.1.0
 	github.com/fluxcd/pkg/kustomize v0.8.0
 	github.com/fluxcd/pkg/runtime v0.20.0
 	github.com/fluxcd/pkg/ssa v0.21.0
-	github.com/fluxcd/pkg/testserver v0.3.0
-	github.com/fluxcd/pkg/untar v0.2.0
+	github.com/fluxcd/pkg/tar v0.1.0
+	github.com/fluxcd/pkg/testserver v0.4.0
 	github.com/fluxcd/source-controller/api v0.30.0
-	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/hashicorp/vault/api v1.8.0
-	github.com/onsi/gomega v1.20.2
+	github.com/onsi/gomega v1.21.1
 	github.com/ory/dockertest v3.3.5+incompatible
 	github.com/otiai10/copy v1.7.0
 	github.com/spf13/pflag v1.0.5
@@ -150,6 +150,7 @@ require (
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-plugin v1.4.3 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/mlock v0.1.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect