gitrepo: Add support for specifying proxy per GitRepository

Add `.spec.proxy.secretRef.name` to `GitRepository` to allow referencing
a secret containing the proxy settings to be used for all remote Git
operations for a particular object.

Signed-off-by: Sanskar Jaiswal <jaiswalsanskar078@gmail.com>

api/v1/gitrepository_types.go

@@ -78,6 +78,10 @@ type GitRepositorySpec struct {
 	// +optional
 	Verification *GitRepositoryVerification `json:"verify,omitempty"`
 
+	// ProxySecretRef specifies the Secret containing the proxy configuration
+	// to use while communicating with the Git server.
+	ProxySecretRef *meta.LocalObjectReference `json:"proxySecretRef,omitempty"`
+
 	// Ignore overrides the set of excluded patterns in the .sourceignore format
 	// (which is the same as .gitignore). If not provided, a default will be used,
 	// consult the documentation for your version to find out what those are.

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -90,6 +90,16 @@ spec:
                 description: Interval at which to check the GitRepository for updates.
                 pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
                 type: string
+              proxySecretRef:
+                description: ProxySecretRef specifies the Secret containing the proxy
+                  configuration to use while communicating with the Git server.
+                properties:
+                  name:
+                    description: Name of the referent.
+                    type: string
+                required:
+                - name
+                type: object
               recurseSubmodules:
                 description: RecurseSubmodules enables the initialization of all submodules
                   within the GitRepository as cloned from the URL, using their default

docs/api/v1/source.md

@@ -157,6 +157,20 @@ signature(s).</p>
 </tr>
 <tr>
 <td>
+<code>proxySecretRef</code><br>
+<em>
+<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
+github.com/fluxcd/pkg/apis/meta.LocalObjectReference
+</a>
+</em>
+</td>
+<td>
+<p>ProxySecretRef specifies the Secret containing the proxy configuration
+to use while communicating with the Git server.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ignore</code><br>
 <em>
 string
@@ -593,6 +607,20 @@ signature(s).</p>
 </tr>
 <tr>
 <td>
+<code>proxySecretRef</code><br>
+<em>
+<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
+github.com/fluxcd/pkg/apis/meta.LocalObjectReference
+</a>
+</em>
+</td>
+<td>
+<p>ProxySecretRef specifies the Secret containing the proxy configuration
+to use while communicating with the Git server.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ignore</code><br>
 <em>
 string

docs/spec/v1/gitrepositories.md

@@ -433,10 +433,53 @@ GitRepository, and changes to the resource or in the Git repository will not
 result in a new Artifact. When the field is set to `false` or removed, it will
 resume.
 
-#### Proxy support
+### Proxy
 
-When a proxy is configured in the source-controller Pod through the appropriate
-environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc.
+`.spec.proxy.secretRef.name` is an optional field to specify a name of a Secret
+containing the proxy settings that need to be used for all remote Git operations
+for the particular GitRepository object. The Secret can contain three keys:
+
+- `address`: The address of the proxy server. This is a required key.
+- `username`: The username to use if the proxy server is protected by basic
+   authentication. This is an optinal key.
+- `password`: The password to use if the proxy server is protected by basic
+   authentication. This is an optinal key.
+
+The proxy server must be either HTTP/S or SOCKS5. You can use a SOCKS5 proxy
+with a HTTP/S Git repository url.
+
+Examples:
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: http-proxy
+type: Opaque
+stringData:
+  address: http://proxy.com
+  username: mandalorian
+  password: grogu
+```
+
+```yaml
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ssh-proxy
+type: Opaque
+stringData:
+  address: socks5://proxy.com
+  username: mandalorian
+  password: grogu
+```
+
+Proxying can also be configured in the source-controller pod directly by using
+the standard environment variables such as `HTTPS_PROXY`, `ALL_PROXY`, etc.
+
+`.spec.proxy.secretRef.name` takes precedence over all environment variables.
 
 ### Recurse submodules
 

go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/fluxcd/pkg/apis/event v0.5.1
 	github.com/fluxcd/pkg/apis/meta v1.1.1
 	github.com/fluxcd/pkg/git v0.12.3
-	github.com/fluxcd/pkg/git/gogit v0.12.0
+	github.com/fluxcd/pkg/git/gogit v0.12.1
 	github.com/fluxcd/pkg/gittestserver v0.8.4
 	github.com/fluxcd/pkg/helmtestserver v0.13.1
 	github.com/fluxcd/pkg/lockedfile v0.1.0

go.sum

@@ -391,8 +391,8 @@ github.com/fluxcd/pkg/apis/meta v1.1.1 h1:sLAKLbEu7rRzJ+Mytffu3NcpfdbOBTa6hcpOQz
 github.com/fluxcd/pkg/apis/meta v1.1.1/go.mod h1:soCfzjFWbm1mqybDcOywWKTCEYlH3skpoNGTboVk234=
 github.com/fluxcd/pkg/git v0.12.3 h1:1KmRYTdcBKDUutg6NIT4x0BCCMT72PjjXs3AnHjybHY=
 github.com/fluxcd/pkg/git v0.12.3/go.mod h1:ID2sry5OqYbgJxvANc7s6V/YwafnQd7e1AGoDvwztAU=
-github.com/fluxcd/pkg/git/gogit v0.12.0 h1:0mCwQND0WpCVZYHLWcXJxRvKVcyWxH4JjMQFMaea8Q4=
-github.com/fluxcd/pkg/git/gogit v0.12.0/go.mod h1:Kn+GfYfZBBIaXmQj39cQvrDxT/6y8leQxXZ5/B+YYTQ=
+github.com/fluxcd/pkg/git/gogit v0.12.1 h1:06jzHOTntYN5xCSQvyFXtLXdqoP8crLh7VYgtXS9+wo=
+github.com/fluxcd/pkg/git/gogit v0.12.1/go.mod h1:Z4Ysp8VifKTvWpjJMKncJsgb2iBqHuIeK80VGjlU41Y=
 github.com/fluxcd/pkg/gittestserver v0.8.4 h1:rA/QUZnfH77ZZG+5xfMqjgEHJdzeeE6Nn1o8cops/bU=
 github.com/fluxcd/pkg/gittestserver v0.8.4/go.mod h1:i3Vng3Stl5zOuGhN4+RuP2NWf5snJCeGUKA7pzAvcHU=
 github.com/fluxcd/pkg/helmtestserver v0.13.1 h1:SjEk9QaMWMjwnqTXGtfMeorC5H+KDvV2YK87Sr2dFng=

internal/controller/gitrepository_controller.go

@@ -28,6 +28,7 @@ import (
 
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/fluxcd/pkg/runtime/logger"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -473,24 +474,35 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 		conditions.Delete(obj, sourcev1.SourceVerifiedCondition)
 	}
 
+	var proxyOpts *transport.ProxyOptions
+	if obj.Spec.ProxySecretRef != nil {
+		var err error
+		proxyOpts, err = r.getProxyOpts(ctx, obj.Spec.ProxySecretRef.Name, obj.GetNamespace())
+		if err != nil {
+			e := serror.NewGeneric(
+				fmt.Errorf("failed to configure proxy options: %w", err),
+				sourcev1.AuthenticationFailedReason,
+			)
+			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
+			// Return error as the world as observed may change
+			return sreconcile.ResultEmpty, e
+		}
+	}
+
 	var authData map[string][]byte
 	if obj.Spec.SecretRef != nil {
 		// Attempt to retrieve secret
-		name := types.NamespacedName{
-			Namespace: obj.GetNamespace(),
-			Name:      obj.Spec.SecretRef.Name,
-		}
-		var secret corev1.Secret
-		if err := r.Client.Get(ctx, name, &secret); err != nil {
+		var err error
+		authData, err = r.getSecretData(ctx, obj.Spec.SecretRef.Name, obj.GetNamespace())
+		if err != nil {
 			e := serror.NewGeneric(
-				fmt.Errorf("failed to get secret '%s': %w", name.String(), err),
+				fmt.Errorf("failed to get secret '%s/%s': %w", obj.GetNamespace(), obj.Spec.SecretRef.Name, err),
 				sourcev1.AuthenticationFailedReason,
 			)
 			conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
 			// Return error as the world as observed may change
 			return sreconcile.ResultEmpty, e
 		}
-		authData = secret.Data
 	}
 
 	u, err := url.Parse(obj.Spec.URL)
@@ -536,7 +548,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 	// Persist the ArtifactSet.
 	*includes = *artifacts
 
-	c, err := r.gitCheckout(ctx, obj, authOpts, dir, true)
+	c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, true)
 	if err != nil {
 		return sreconcile.ResultEmpty, err
 	}
@@ -578,7 +590,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 
 		// If we can't skip the reconciliation, checkout again without any
 		// optimization.
-		c, err := r.gitCheckout(ctx, obj, authOpts, dir, false)
+		c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, false)
 		if err != nil {
 			return sreconcile.ResultEmpty, err
 		}
@@ -606,6 +618,39 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch
 	return sreconcile.ResultSuccess, nil
 }
 
+// getProxyOpts fetches the secret containing the proxy settings, constructs a
+// transport.ProxyOptions object using those settings and then returns it.
+func (r *GitRepositoryReconciler) getProxyOpts(ctx context.Context, proxySecretName,
+	proxySecretNamespace string) (*transport.ProxyOptions, error) {
+	proxyData, err := r.getSecretData(ctx, proxySecretName, proxySecretNamespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get secret '%s/%s': %w", proxySecretNamespace, proxySecretName, err)
+	}
+	address, ok := proxyData["address"]
+	if !ok {
+		return nil, fmt.Errorf("invalid proxy secret '%s/%s': key 'address' is missing", proxySecretNamespace, proxySecretName)
+	}
+
+	proxyOpts := &transport.ProxyOptions{
+		URL:      string(address),
+		Username: string(proxyData["username"]),
+		Password: string(proxyData["password"]),
+	}
+	return proxyOpts, nil
+}
+
+func (r *GitRepositoryReconciler) getSecretData(ctx context.Context, name, namespace string) (map[string][]byte, error) {
+	key := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	var secret corev1.Secret
+	if err := r.Client.Get(ctx, key, &secret); err != nil {
+		return nil, err
+	}
+	return secret.Data, nil
+}
+
 // reconcileArtifact archives a new Artifact to the Storage, if the current
 // (Status) data on the object does not match the given.
 //
@@ -776,8 +821,8 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, sp *patc
 
 // gitCheckout builds checkout options with the given configurations and
 // performs a git checkout.
-func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
-	obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string, optimized bool) (*git.Commit, error) {
+func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, obj *sourcev1.GitRepository,
+	authOpts *git.AuthOptions, proxyOpts *transport.ProxyOptions, dir string, optimized bool) (*git.Commit, error) {
 	// Configure checkout strategy.
 	cloneOpts := repository.CloneConfig{
 		RecurseSubmodules: obj.Spec.RecurseSubmodules,
@@ -807,6 +852,9 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
 	if authOpts.Transport == git.HTTP {
 		clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
 	}
+	if proxyOpts != nil {
+		clientOpts = append(clientOpts, gogit.WithProxy(*proxyOpts))
+	}
 
 	gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...)
 	if err != nil {

internal/controller/gitrepository_controller_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/storage/memory"
 	. "github.com/onsi/gomega"
 	sshtestdata "golang.org/x/crypto/ssh/testdata"
@@ -1624,6 +1625,76 @@ func TestGitRepositoryReconciler_verifyCommitSignature(t *testing.T) {
 	}
 }
 
+func TestGitRepositoryReconciler_getProxyOpts(t *testing.T) {
+	invalidProxy := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "invalid-proxy",
+		},
+		Data: map[string][]byte{
+			"url": []byte("https://example.com"),
+		},
+	}
+	validProxy := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "valid-proxy",
+		},
+		Data: map[string][]byte{
+			"address":  []byte("https://example.com"),
+			"username": []byte("user"),
+			"password": []byte("pass"),
+		},
+	}
+
+	clientBuilder := fakeclient.NewClientBuilder().
+		WithScheme(testEnv.GetScheme())
+	clientBuilder.WithObjects(invalidProxy, validProxy)
+
+	r := &GitRepositoryReconciler{
+		Client: clientBuilder.Build(),
+	}
+
+	tests := []struct {
+		name      string
+		secret    string
+		err       string
+		proxyOpts *transport.ProxyOptions
+	}{
+		{
+			name:   "non-existent secret",
+			secret: "non-existent",
+			err:    "failed to get secret '/non-existent': ",
+		},
+		{
+			name:   "invalid proxy secret",
+			secret: "invalid-proxy",
+			err:    "invalid proxy secret '/invalid-proxy': key 'address' is missing",
+		},
+		{
+			name:   "valid proxy secret",
+			secret: "valid-proxy",
+			proxyOpts: &transport.ProxyOptions{
+				URL:      "https://example.com",
+				Username: "user",
+				Password: "pass",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			opts, err := r.getProxyOpts(context.TODO(), tt.secret, "")
+			if opts != nil {
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(opts).To(Equal(tt.proxyOpts))
+			} else {
+				g.Expect(err).To(HaveOccurred())
+				g.Expect(err.Error()).To(ContainSubstring(tt.err))
+			}
+		})
+	}
+}
+
 func TestGitRepositoryReconciler_ConditionsUpdate(t *testing.T) {
 	g := NewWithT(t)