Merge pull request #776 from vmware-tanzu/pkg-repo-sidecar-v038x

execute pkg repo fetching in the sidecar (v0.38.x backport)

cmd/controller/run.go

@@ -130,6 +130,8 @@ func Run(opts Options, runLog logr.Logger) error {
 		return fmt.Errorf("Starting RPC client: %s", err)
 	}
 
+	sidecarCmdExec := sidecarClient.CmdExec()
+
 	{ // add controller for config
 		reconciler := kcconfig.NewReconciler(
 			coreClient, sidecarClient.OSConfig(), runLog.WithName("config"))
@@ -169,7 +171,7 @@ func Run(opts Options, runLog logr.Logger) error {
 			AppClient:  kcClient,
 			KcConfig:   kcConfig,
 			AppMetrics: appMetrics,
-			CmdRunner:  sidecarClient.CmdExec(),
+			CmdRunner:  sidecarCmdExec,
 		}
 		reconciler := app.NewReconciler(kcClient, runLog.WithName("app"),
 			appFactory, refTracker, updateStatusTracker)
@@ -213,7 +215,7 @@ func Run(opts Options, runLog logr.Logger) error {
 	}
 
 	{ // add controller for pkgrepositories
-		appFactory := pkgrepository.AppFactory{coreClient, kcClient, kcConfig}
+		appFactory := pkgrepository.AppFactory{coreClient, kcClient, kcConfig, sidecarCmdExec}
 
 		reconciler := pkgrepository.NewReconciler(kcClient, coreClient,
 			runLog.WithName("pkgr"), appFactory, refTracker, updateStatusTracker)

pkg/pkgrepository/app_factory.go

@@ -21,13 +21,13 @@ type AppFactory struct {
 	CoreClient kubernetes.Interface
 	AppClient  kcclient.Interface
 	KcConfig   *config.Config
+	CmdRunner  exec.CmdRunner
 }
 
 // NewCRDPackageRepo constructs "hidden" App to reconcile PackageRepository.
 func (f *AppFactory) NewCRDPackageRepo(app *kcv1alpha1.App, pkgr *pkgv1alpha1.PackageRepository, log logr.Logger) *CRDApp {
-	cmdRunner := exec.PlainCmdRunner{}
-	fetchFactory := fetch.NewFactory(f.CoreClient, fetch.VendirOpts{SkipTLSConfig: f.KcConfig}, cmdRunner)
-	templateFactory := template.NewFactory(f.CoreClient, fetchFactory, false, cmdRunner)
-	deployFactory := deploy.NewFactory(f.CoreClient, nil, cmdRunner, log)
+	fetchFactory := fetch.NewFactory(f.CoreClient, fetch.VendirOpts{SkipTLSConfig: f.KcConfig}, f.CmdRunner)
+	templateFactory := template.NewFactory(f.CoreClient, fetchFactory, false, f.CmdRunner)
+	deployFactory := deploy.NewFactory(f.CoreClient, nil, f.CmdRunner, log)
 	return NewCRDApp(app, pkgr, log, f.AppClient, fetchFactory, templateFactory, deployFactory)
 }

pkg/pkgrepository/app_fetch.go

@@ -6,14 +6,11 @@ package pkgrepository
 import (
 	"bytes"
 	"fmt"
-	"os"
 	"path"
 	"strconv"
 	"time"
 
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/exec"
-
-	goexec "os/exec"
 )
 
 func (a *App) fetch(dstPath string) (string, exec.CmdRunResult) {
@@ -42,7 +39,8 @@ func (a *App) fetch(dstPath string) (string, exec.CmdRunResult) {
 		return "", result
 	}
 
-	result = a.runVendir(conf, dstPath)
+	result = vendir.Run(conf, dstPath)
+
 	// retry if error occurs before reporting failure.
 	// This is mainly done to support private registry
 	// authentication for images/bundles since placeholder
@@ -63,7 +61,7 @@ func (a *App) fetch(dstPath string) (string, exec.CmdRunResult) {
 				// no secrets/configmaps have changed, no point in retrying
 				continue
 			}
-			result = a.runVendir(newConf, dstPath)
+			result = vendir.Run(newConf, dstPath)
 			if result.Error == nil {
 				break
 			}
@@ -80,22 +78,3 @@ func (a *App) fetch(dstPath string) (string, exec.CmdRunResult) {
 
 	return dstPath, result
 }
-
-func (a *App) runVendir(conf []byte, workingDir string) exec.CmdRunResult {
-	var stdoutBs, stderrBs bytes.Buffer
-	cmd := goexec.Command("vendir", "sync", "-f", "-", "--lock-file", os.DevNull)
-	cmd.Dir = workingDir
-	cmd.Stdin = bytes.NewReader(conf)
-	cmd.Stdout = &stdoutBs
-	cmd.Stderr = &stderrBs
-
-	err := cmd.Run()
-
-	result := exec.CmdRunResult{
-		Stdout: stdoutBs.String(),
-		Stderr: stderrBs.String(),
-	}
-	result.AttachErrorf("Fetching resources: %s", err)
-
-	return result
-}

test/e2e/assets/https-server/server.yml

@@ -132,6 +132,26 @@ data:
       name: http-server-returned-cm
     data:
       content: http-server-returned-content
+binaryData:
+  # Includes single file: packages/package.yml
+  #   ---
+  #   apiVersion: data.packaging.carvel.dev/v1alpha1
+  #   kind: Package
+  #   metadata:
+  #     name: package-behind-ca-cert.carvel.dev.1.0.0
+  #   spec:
+  #     refName: package-behind-ca-cert.carvel.dev
+  #     version: 1.0.0
+  #     template:
+  #       spec:
+  #         fetch:
+  #         - http:
+  #             url: unused
+  #         template:
+  #         - ytt: {}
+  #         deploy:
+  #         - kapp: {}
+  packages.tar: "H4sIAHczxmIAA+2SS2rEMAyGZ51T6AL22KmTQg5RuupetZVJyGOM4wRC6d0bx5M+oAylDJRCvo0sS/+PjGVRN3ii4Wjjgc9de7gxQohcKQjxPs/WKNKYr2QiA6nSXKo7makUhFSZzA4gbj3Id4yDR7eMYprrfUtbWV6px6fAe/wnMMYStPUTuaE+9wUY9MjjLtT9iWt0E7Xc0HScJLa2Qpk0dW8KeIz7knTkMYiKBKDHjgq4bBJ7pmrpZBqZJuc/WXHJBRfJYEkHlaPy4WfCpXnaBo0eAJ4626Kn4ASweQZK8rraEgaV93bLAqNrCxj7cSBzuf1qFTSz9wW8vF5yQ7Y9zx/VBq1dy3/9iTs7Ozu/4A1GdDfIAAgAAA=="
 
 # TODO should we make vendir's http retry within App CR, to avoid
 # transient failure when Service=>Deployment networking is not ready?

test/e2e/kappcontroller/http_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/vmware-tanzu/carvel-kapp-controller/pkg/apis/kappctrl/v1alpha1"
 	"github.com/vmware-tanzu/carvel-kapp-controller/test/e2e"
+	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
@@ -128,6 +129,7 @@ func TestHTTPSSelfSignedCerts(t *testing.T) {
 	env := e2e.BuildEnv(t)
 	logger := e2e.Logger{}
 	kapp := e2e.Kapp{t, env.Namespace, logger}
+	kubectl := e2e.Kubectl{t, env.Namespace, logger}
 	sas := e2e.ServiceAccounts{env.Namespace}
 
 	// When updating, certs and keys must be regenerated for server and added to server.go and config-test/config-map.yml
@@ -154,10 +156,12 @@ spec:
 `, serverNamespace, env.Namespace) + sas.ForNamespaceYAML()
 
 	name := "test-https"
+	pkgrName := "test-https-pkgr"
 	httpsServerName := "test-https-server"
 
 	cleanUp := func() {
 		kapp.Run([]string{"delete", "-a", name})
+		kapp.Run([]string{"delete", "-a", pkgrName})
 		kapp.Run([]string{"delete", "-a", httpsServerName, "-n", serverNamespace})
 	}
 
@@ -240,4 +244,27 @@ spec:
 		}
 	})
 
+	logger.Section("deploy package repository that fetches content from http server", func() {
+		pkgrConfig := `---
+apiVersion: packaging.carvel.dev/v1alpha1
+kind: PackageRepository
+metadata:
+  name: test-https-pkgr
+spec:
+  fetch:
+    http:
+      # use https to exercise CA certificate validation
+      # When updating address, certs and keys must be regenerated
+      # for server and added to e2e/assets/https-server
+      url: https://https-svc.https-server.svc.cluster.local:443/packages.tar
+`
+
+		kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", pkgrName}, e2e.RunOpts{
+			StdinReader: strings.NewReader(pkgrConfig),
+			OnErrKubectl: []string{"get", "pkgr/test-https-pkgr", "-oyaml"},
+		})
+
+		out := kubectl.Run([]string{"get", "pkg", "package-behind-ca-cert.carvel.dev.1.0.0", "-oyaml"})
+		assert.Contains(t, out, "name: package-behind-ca-cert.carvel.dev.1.0.0\n")
+	})
 }