From d23aa68320ccc7c453c2cc357dc9c290e8350d93 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Tue, 30 May 2023 17:54:58 +0530 Subject: [PATCH] gitrepo: Add support for specifying proxy per `GitRepository` Add `.spec.proxy.secretRef.name` to `GitRepository` to allow referencing a secret containing the proxy settings to be used for all remote Git operations for a particular object. Signed-off-by: Sanskar Jaiswal --- api/v1/gitrepository_types.go | 4 ++ api/v1/zz_generated.deepcopy.go | 5 ++ ...rce.toolkit.fluxcd.io_gitrepositories.yaml | 10 +++ docs/api/v1/source.md | 28 ++++++++ docs/spec/v1/gitrepositories.md | 49 ++++++++++++- go.mod | 2 +- go.sum | 4 +- .../controller/gitrepository_controller.go | 72 +++++++++++++++---- .../gitrepository_controller_test.go | 71 ++++++++++++++++++ 9 files changed, 227 insertions(+), 18 deletions(-) diff --git a/api/v1/gitrepository_types.go b/api/v1/gitrepository_types.go index 4475acba4..0ae158560 100644 --- a/api/v1/gitrepository_types.go +++ b/api/v1/gitrepository_types.go @@ -78,6 +78,10 @@ type GitRepositorySpec struct { // +optional Verification *GitRepositoryVerification `json:"verify,omitempty"` + // ProxySecretRef specifies the Secret containing the proxy configuration + // to use while communicating with the Git server. + ProxySecretRef *meta.LocalObjectReference `json:"proxySecretRef,omitempty"` + // Ignore overrides the set of excluded patterns in the .sourceignore format // (which is the same as .gitignore). If not provided, a default will be used, // consult the documentation for your version to find out what those are. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 0b0fde694..23630ff9f 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -169,6 +169,11 @@ func (in *GitRepositorySpec) DeepCopyInto(out *GitRepositorySpec) { *out = new(GitRepositoryVerification) **out = **in } + if in.ProxySecretRef != nil { + in, out := &in.ProxySecretRef, &out.ProxySecretRef + *out = new(meta.LocalObjectReference) + **out = **in + } if in.Ignore != nil { in, out := &in.Ignore, &out.Ignore *out = new(string) diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 3097292ca..ba19ecd05 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -90,6 +90,16 @@ spec: description: Interval at which to check the GitRepository for updates. pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ type: string + proxySecretRef: + description: ProxySecretRef specifies the Secret containing the proxy + configuration to use while communicating with the Git server. + properties: + name: + description: Name of the referent. + type: string + required: + - name + type: object recurseSubmodules: description: RecurseSubmodules enables the initialization of all submodules within the GitRepository as cloned from the URL, using their default diff --git a/docs/api/v1/source.md b/docs/api/v1/source.md index ed4862bbe..5771a2b30 100644 --- a/docs/api/v1/source.md +++ b/docs/api/v1/source.md @@ -157,6 +157,20 @@ signature(s).

+proxySecretRef
+ + +github.com/fluxcd/pkg/apis/meta.LocalObjectReference + + + + +

ProxySecretRef specifies the Secret containing the proxy configuration +to use while communicating with the Git server.

+ + + + ignore
string @@ -593,6 +607,20 @@ signature(s).

+proxySecretRef
+ + +github.com/fluxcd/pkg/apis/meta.LocalObjectReference + + + + +

ProxySecretRef specifies the Secret containing the proxy configuration +to use while communicating with the Git server.

+ + + + ignore
string diff --git a/docs/spec/v1/gitrepositories.md b/docs/spec/v1/gitrepositories.md index 4e755973c..5d2caacff 100644 --- a/docs/spec/v1/gitrepositories.md +++ b/docs/spec/v1/gitrepositories.md @@ -433,10 +433,53 @@ GitRepository, and changes to the resource or in the Git repository will not result in a new Artifact. When the field is set to `false` or removed, it will resume. -#### Proxy support +### Proxy -When a proxy is configured in the source-controller Pod through the appropriate -environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc. +`.spec.proxy.secretRef.name` is an optional field to specify a name of a Secret +containing the proxy settings that need to be used for all remote Git operations +for the particular GitRepository object. The Secret can contain three keys: + +- `address`: The address of the proxy server. This is a required key. +- `username`: The username to use if the proxy server is protected by basic + authentication. This is an optinal key. +- `password`: The password to use if the proxy server is protected by basic + authentication. This is an optinal key. + +The proxy server must be either HTTP/S or SOCKS5. You can use a SOCKS5 proxy +with a HTTP/S Git repository url. + +Examples: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: http-proxy +type: Opaque +stringData: + address: http://proxy.com + username: mandalorian + password: grogu +``` + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: ssh-proxy +type: Opaque +stringData: + address: socks5://proxy.com + username: mandalorian + password: grogu +``` + +Proxying can also be configured in the source-controller pod directly by using +the standard environment variables such as `HTTPS_PROXY`, `ALL_PROXY`, etc. + +`.spec.proxy.secretRef.name` takes precedence over all environment variables. ### Recurse submodules diff --git a/go.mod b/go.mod index 0336f9937..18f21afd4 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/fluxcd/pkg/apis/event v0.5.1 github.com/fluxcd/pkg/apis/meta v1.1.1 github.com/fluxcd/pkg/git v0.12.3 - github.com/fluxcd/pkg/git/gogit v0.12.0 + github.com/fluxcd/pkg/git/gogit v0.12.1 github.com/fluxcd/pkg/gittestserver v0.8.4 github.com/fluxcd/pkg/helmtestserver v0.13.1 github.com/fluxcd/pkg/lockedfile v0.1.0 diff --git a/go.sum b/go.sum index 4e2c8e085..a0424738c 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/fluxcd/pkg/apis/meta v1.1.1 h1:sLAKLbEu7rRzJ+Mytffu3NcpfdbOBTa6hcpOQz github.com/fluxcd/pkg/apis/meta v1.1.1/go.mod h1:soCfzjFWbm1mqybDcOywWKTCEYlH3skpoNGTboVk234= github.com/fluxcd/pkg/git v0.12.3 h1:1KmRYTdcBKDUutg6NIT4x0BCCMT72PjjXs3AnHjybHY= github.com/fluxcd/pkg/git v0.12.3/go.mod h1:ID2sry5OqYbgJxvANc7s6V/YwafnQd7e1AGoDvwztAU= -github.com/fluxcd/pkg/git/gogit v0.12.0 h1:0mCwQND0WpCVZYHLWcXJxRvKVcyWxH4JjMQFMaea8Q4= -github.com/fluxcd/pkg/git/gogit v0.12.0/go.mod h1:Kn+GfYfZBBIaXmQj39cQvrDxT/6y8leQxXZ5/B+YYTQ= +github.com/fluxcd/pkg/git/gogit v0.12.1 h1:06jzHOTntYN5xCSQvyFXtLXdqoP8crLh7VYgtXS9+wo= +github.com/fluxcd/pkg/git/gogit v0.12.1/go.mod h1:Z4Ysp8VifKTvWpjJMKncJsgb2iBqHuIeK80VGjlU41Y= github.com/fluxcd/pkg/gittestserver v0.8.4 h1:rA/QUZnfH77ZZG+5xfMqjgEHJdzeeE6Nn1o8cops/bU= github.com/fluxcd/pkg/gittestserver v0.8.4/go.mod h1:i3Vng3Stl5zOuGhN4+RuP2NWf5snJCeGUKA7pzAvcHU= github.com/fluxcd/pkg/helmtestserver v0.13.1 h1:SjEk9QaMWMjwnqTXGtfMeorC5H+KDvV2YK87Sr2dFng= diff --git a/internal/controller/gitrepository_controller.go b/internal/controller/gitrepository_controller.go index d620c82fb..3e71f05ed 100644 --- a/internal/controller/gitrepository_controller.go +++ b/internal/controller/gitrepository_controller.go @@ -28,6 +28,7 @@ import ( securejoin "github.com/cyphar/filepath-securejoin" "github.com/fluxcd/pkg/runtime/logger" + "github.com/go-git/go-git/v5/plumbing/transport" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -473,24 +474,35 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch conditions.Delete(obj, sourcev1.SourceVerifiedCondition) } + var proxyOpts *transport.ProxyOptions + if obj.Spec.ProxySecretRef != nil { + var err error + proxyOpts, err = r.getProxyOpts(ctx, obj.Spec.ProxySecretRef.Name, obj.GetNamespace()) + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to configure proxy options: %w", err), + sourcev1.AuthenticationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + // Return error as the world as observed may change + return sreconcile.ResultEmpty, e + } + } + var authData map[string][]byte if obj.Spec.SecretRef != nil { // Attempt to retrieve secret - name := types.NamespacedName{ - Namespace: obj.GetNamespace(), - Name: obj.Spec.SecretRef.Name, - } - var secret corev1.Secret - if err := r.Client.Get(ctx, name, &secret); err != nil { + var err error + authData, err = r.getSecretData(ctx, obj.Spec.SecretRef.Name, obj.GetNamespace()) + if err != nil { e := serror.NewGeneric( - fmt.Errorf("failed to get secret '%s': %w", name.String(), err), + fmt.Errorf("failed to get secret '%s/%s': %w", obj.GetNamespace(), obj.Spec.SecretRef.Name, err), sourcev1.AuthenticationFailedReason, ) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) // Return error as the world as observed may change return sreconcile.ResultEmpty, e } - authData = secret.Data } u, err := url.Parse(obj.Spec.URL) @@ -536,7 +548,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch // Persist the ArtifactSet. *includes = *artifacts - c, err := r.gitCheckout(ctx, obj, authOpts, dir, true) + c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, true) if err != nil { return sreconcile.ResultEmpty, err } @@ -578,7 +590,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch // If we can't skip the reconciliation, checkout again without any // optimization. - c, err := r.gitCheckout(ctx, obj, authOpts, dir, false) + c, err := r.gitCheckout(ctx, obj, authOpts, proxyOpts, dir, false) if err != nil { return sreconcile.ResultEmpty, err } @@ -606,6 +618,39 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, sp *patch return sreconcile.ResultSuccess, nil } +// getProxyOpts fetches the secret containing the proxy settings, constructs a +// transport.ProxyOptions object using those settings and then returns it. +func (r *GitRepositoryReconciler) getProxyOpts(ctx context.Context, proxySecretName, + proxySecretNamespace string) (*transport.ProxyOptions, error) { + proxyData, err := r.getSecretData(ctx, proxySecretName, proxySecretNamespace) + if err != nil { + return nil, fmt.Errorf("failed to get secret '%s/%s': %w", proxySecretNamespace, proxySecretName, err) + } + address, ok := proxyData["address"] + if !ok { + return nil, fmt.Errorf("invalid proxy secret '%s/%s': key 'address' is missing", proxySecretNamespace, proxySecretName) + } + + proxyOpts := &transport.ProxyOptions{ + URL: string(address), + Username: string(proxyData["username"]), + Password: string(proxyData["password"]), + } + return proxyOpts, nil +} + +func (r *GitRepositoryReconciler) getSecretData(ctx context.Context, name, namespace string) (map[string][]byte, error) { + key := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + var secret corev1.Secret + if err := r.Client.Get(ctx, key, &secret); err != nil { + return nil, err + } + return secret.Data, nil +} + // reconcileArtifact archives a new Artifact to the Storage, if the current // (Status) data on the object does not match the given. // @@ -776,8 +821,8 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, sp *patc // gitCheckout builds checkout options with the given configurations and // performs a git checkout. -func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, - obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string, optimized bool) (*git.Commit, error) { +func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, obj *sourcev1.GitRepository, + authOpts *git.AuthOptions, proxyOpts *transport.ProxyOptions, dir string, optimized bool) (*git.Commit, error) { // Configure checkout strategy. cloneOpts := repository.CloneConfig{ RecurseSubmodules: obj.Spec.RecurseSubmodules, @@ -807,6 +852,9 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, if authOpts.Transport == git.HTTP { clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP()) } + if proxyOpts != nil { + clientOpts = append(clientOpts, gogit.WithProxy(*proxyOpts)) + } gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...) if err != nil { diff --git a/internal/controller/gitrepository_controller_test.go b/internal/controller/gitrepository_controller_test.go index 62c90b6d5..5a1ba18a4 100644 --- a/internal/controller/gitrepository_controller_test.go +++ b/internal/controller/gitrepository_controller_test.go @@ -33,6 +33,7 @@ import ( "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/memory" . "github.com/onsi/gomega" sshtestdata "golang.org/x/crypto/ssh/testdata" @@ -1624,6 +1625,76 @@ func TestGitRepositoryReconciler_verifyCommitSignature(t *testing.T) { } } +func TestGitRepositoryReconciler_getProxyOpts(t *testing.T) { + invalidProxy := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-proxy", + }, + Data: map[string][]byte{ + "url": []byte("https://example.com"), + }, + } + validProxy := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "valid-proxy", + }, + Data: map[string][]byte{ + "address": []byte("https://example.com"), + "username": []byte("user"), + "password": []byte("pass"), + }, + } + + clientBuilder := fakeclient.NewClientBuilder(). + WithScheme(testEnv.GetScheme()) + clientBuilder.WithObjects(invalidProxy, validProxy) + + r := &GitRepositoryReconciler{ + Client: clientBuilder.Build(), + } + + tests := []struct { + name string + secret string + err string + proxyOpts *transport.ProxyOptions + }{ + { + name: "non-existent secret", + secret: "non-existent", + err: "failed to get secret '/non-existent': ", + }, + { + name: "invalid proxy secret", + secret: "invalid-proxy", + err: "invalid proxy secret '/invalid-proxy': key 'address' is missing", + }, + { + name: "valid proxy secret", + secret: "valid-proxy", + proxyOpts: &transport.ProxyOptions{ + URL: "https://example.com", + Username: "user", + Password: "pass", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + opts, err := r.getProxyOpts(context.TODO(), tt.secret, "") + if opts != nil { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(opts).To(Equal(tt.proxyOpts)) + } else { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.err)) + } + }) + } +} + func TestGitRepositoryReconciler_ConditionsUpdate(t *testing.T) { g := NewWithT(t)