From 8aaf652a1c2b36b516a90dfe8f69fc846134e999 Mon Sep 17 00:00:00 2001 From: Vadim Rutkovsky Date: Tue, 9 Jul 2024 16:18:14 +0200 Subject: [PATCH] New cert rotation controlers --- pkg/operator/certrotation/cabundle.go | 31 +- .../client_cert_rotation_controller.go | 228 ++++-- .../client_cert_rotation_controllers_test.go | 652 ++++++++++++++++++ pkg/operator/certrotation/signer.go | 8 + pkg/operator/certrotation/status_reporter.go | 36 + 5 files changed, 883 insertions(+), 72 deletions(-) create mode 100644 pkg/operator/certrotation/client_cert_rotation_controllers_test.go create mode 100644 pkg/operator/certrotation/status_reporter.go diff --git a/pkg/operator/certrotation/cabundle.go b/pkg/operator/certrotation/cabundle.go index 58c10b7cad..fc75ff0a87 100644 --- a/pkg/operator/certrotation/cabundle.go +++ b/pkg/operator/certrotation/cabundle.go @@ -41,7 +41,7 @@ type CABundleConfigMap struct { EventRecorder events.Recorder } -func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingCertKeyPair *crypto.CA) ([]*x509.Certificate, error) { +func (c CABundleConfigMap) ensureConfigMapCABundleFromCerts(ctx context.Context, signers []*x509.Certificate) ([]*x509.Certificate, error) { // by this point we have current signing cert/key pair. We now need to make sure that the ca-bundle configmap has this cert and // doesn't have any expired certs modified := false @@ -68,7 +68,7 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC needsMetadataUpdate := c.AdditionalAnnotations.EnsureTLSMetadataUpdate(&caBundleConfigMap.ObjectMeta) modified = needsOwnerUpdate || needsMetadataUpdate || modified - updatedCerts, err := manageCABundleConfigMap(caBundleConfigMap, signingCertKeyPair.Config.Certs[0]) + updatedCerts, err := manageCABundleConfigMap(caBundleConfigMap, signers) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC return nil, err } if updated { - klog.V(2).Infof("Updated ca-bundle.crt configmap %s/%s with:\n%s", certs.CertificateBundleToString(updatedCerts), caBundleConfigMap.Namespace, caBundleConfigMap.Name) + klog.V(2).Infof("Updated ca-bundle.crt configmap %s/%s with:\n%s", caBundleConfigMap.Namespace, caBundleConfigMap.Name, certs.CertificateBundleToString(updatedCerts)) } caBundleConfigMap = actualCABundleConfigMap } @@ -103,9 +103,30 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC return certificates, nil } +func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingCertKeyPair *crypto.CA) ([]*x509.Certificate, error) { + return c.ensureConfigMapCABundleFromCerts(ctx, signingCertKeyPair.Config.Certs) +} + +func (c CABundleConfigMap) getConfigMapCABundle() ([]*x509.Certificate, error) { + caBundleConfigMap, err := c.Lister.ConfigMaps(c.Namespace).Get(c.Name) + if err != nil || apierrors.IsNotFound(err) || caBundleConfigMap == nil { + return nil, err + } + caBundle := caBundleConfigMap.Data["ca-bundle.crt"] + if len(caBundle) == 0 { + return nil, fmt.Errorf("configmap/%s -n%s missing ca-bundle.crt", caBundleConfigMap.Name, caBundleConfigMap.Namespace) + } + certificates, err := cert.ParseCertsPEM([]byte(caBundle)) + if err != nil { + return nil, err + } + + return certificates, nil +} + // manageCABundleConfigMap adds the new certificate to the list of cabundles, eliminates duplicates, and prunes the list of expired // certs to trust as signers -func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, currentSigner *x509.Certificate) ([]*x509.Certificate, error) { +func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, signers []*x509.Certificate) ([]*x509.Certificate, error) { if caBundleConfigMap.Data == nil { caBundleConfigMap.Data = map[string]string{} } @@ -119,7 +140,7 @@ func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, currentSigner return nil, err } } - certificates = append([]*x509.Certificate{currentSigner}, certificates...) + certificates = append(signers, certificates...) certificates = crypto.FilterExpiredCerts(certificates...) finalCertificates := []*x509.Certificate{} diff --git a/pkg/operator/certrotation/client_cert_rotation_controller.go b/pkg/operator/certrotation/client_cert_rotation_controller.go index 5159f562a3..409ace01d2 100644 --- a/pkg/operator/certrotation/client_cert_rotation_controller.go +++ b/pkg/operator/certrotation/client_cert_rotation_controller.go @@ -2,16 +2,15 @@ package certrotation import ( "context" + "crypto/x509" "fmt" "time" - operatorv1 "github.com/openshift/api/operator/v1" + "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "github.com/openshift/library-go/pkg/controller/factory" - "github.com/openshift/library-go/pkg/operator/condition" "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/v1helpers" ) const ( @@ -27,82 +26,123 @@ const ( RunOnceContextKey = "cert-rotation-controller.openshift.io/run-once" ) -// StatusReporter knows how to report the status of cert rotation -type StatusReporter interface { - Report(ctx context.Context, controllerName string, syncErr error) (updated bool, updateErr error) +// RotatedSigningCASecretController continuously creates a self-signed signing CA (via RotatedSigningCASecret) and store it in a secret. +type RotatedSigningCASecretController struct { + name string + + // Signer rotates a self-signed signing CA stored in a secret. + Signer *RotatedSigningCASecret + // Plumbing: + StatusReporter StatusReporter } -var _ StatusReporter = (*StaticPodConditionStatusReporter)(nil) +func NewRotatedSigningCASecretController( + signer *RotatedSigningCASecret, + recorder events.Recorder, + reporter StatusReporter, +) factory.Controller { + name := fmt.Sprintf("signer %s/%s", signer.Namespace, signer.Name) + c := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: reporter, + name: name, + } + return factory.New(). + ResyncEvery(time.Minute). + WithSync(c.Sync). + WithInformers( + signer.Informer.Informer(), + ). + ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name)) +} -type StaticPodConditionStatusReporter struct { - // Plumbing: - OperatorClient v1helpers.StaticPodOperatorClient +func (c RotatedSigningCASecretController) SyncWorker(ctx context.Context, syncCtx factory.SyncContext) error { + _, _, err := c.Signer.EnsureSigningCertKeyPair(ctx) + return err } -func (s *StaticPodConditionStatusReporter) Report(ctx context.Context, controllerName string, syncErr error) (bool, error) { - newCondition := operatorv1.OperatorCondition{ - Type: fmt.Sprintf(condition.CertRotationDegradedConditionTypeFmt, controllerName), - Status: operatorv1.ConditionFalse, +func (c RotatedSigningCASecretController) Sync(ctx context.Context, syncCtx factory.SyncContext) error { + syncErr := c.SyncWorker(ctx, syncCtx) + + // running this function with RunOnceContextKey value context will make this "run-once" without updating status. + isRunOnce, ok := ctx.Value(RunOnceContextKey).(bool) + if ok && isRunOnce { + return syncErr } - if syncErr != nil { - newCondition.Status = operatorv1.ConditionTrue - newCondition.Reason = "RotationError" - newCondition.Message = syncErr.Error() + + updated, updateErr := c.StatusReporter.Report(ctx, c.name, syncErr) + if updateErr != nil { + return updateErr + } + if updated && syncErr != nil { + syncCtx.Recorder().Warningf("RotationError", syncErr.Error()) } - _, updated, updateErr := v1helpers.UpdateStaticPodStatus(ctx, s.OperatorClient, v1helpers.UpdateStaticPodConditionFn(newCondition)) - return updated, updateErr + + return syncErr } -// CertRotationController does: -// -// 1) continuously create a self-signed signing CA (via RotatedSigningCASecret) and store it in a secret. -// 2) maintain a CA bundle ConfigMap with all not yet expired CA certs. -// 3) continuously create a target cert and key signed by the latest signing CA and store it in a secret. -type CertRotationController struct { - // controller name - Name string - // RotatedSigningCASecret rotates a self-signed signing CA stored in a secret. - RotatedSigningCASecret RotatedSigningCASecret - // CABundleConfigMap maintains a CA bundle config map, by adding new CA certs coming from rotatedSigningCASecret, and by removing expired old ones. - CABundleConfigMap CABundleConfigMap - // RotatedSelfSignedCertKeySecret rotates a key and cert signed by a signing CA and stores it in a secret. - RotatedSelfSignedCertKeySecret RotatedSelfSignedCertKeySecret +// RotatedCABundleController maintains a CA bundle ConfigMap with all not yet expired CA certs. +type RotatedCABundleController struct { + name string + CABundle *CABundleConfigMap + Signers []*RotatedSigningCASecret // Plumbing: StatusReporter StatusReporter } -func NewCertRotationController( - name string, - rotatedSigningCASecret RotatedSigningCASecret, - caBundleConfigMap CABundleConfigMap, - rotatedSelfSignedCertKeySecret RotatedSelfSignedCertKeySecret, +func NewRotatedCABundleConfigMapController( + cabundle *CABundleConfigMap, + signers []*RotatedSigningCASecret, recorder events.Recorder, reporter StatusReporter, ) factory.Controller { - c := &CertRotationController{ - Name: name, - RotatedSigningCASecret: rotatedSigningCASecret, - CABundleConfigMap: caBundleConfigMap, - RotatedSelfSignedCertKeySecret: rotatedSelfSignedCertKeySecret, - StatusReporter: reporter, + name := fmt.Sprintf("cabundle %s/%s", cabundle.Namespace, cabundle.Name) + c := &RotatedCABundleController{ + CABundle: cabundle, + Signers: signers, + StatusReporter: reporter, + name: name, } - return factory.New(). + ctrlFactory := factory.New(). ResyncEvery(time.Minute). WithSync(c.Sync). WithInformers( - rotatedSigningCASecret.Informer.Informer(), - caBundleConfigMap.Informer.Informer(), - rotatedSelfSignedCertKeySecret.Informer.Informer(), - ). - WithPostStartHooks( - c.targetCertRecheckerPostRunHook, - ). + cabundle.Informer.Informer(), + ) + for _, signer := range signers { + ctrlFactory = ctrlFactory.WithInformers(signer.Informer.Informer()) + } + + return ctrlFactory. ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name)) } -func (c CertRotationController) Sync(ctx context.Context, syncCtx factory.SyncContext) error { - syncErr := c.SyncWorker(ctx) +func (c RotatedCABundleController) SyncWorker(ctx context.Context, syncCtx factory.SyncContext) error { + var errs []error + var signers []*x509.Certificate + for _, signer := range c.Signers { + signingCertKeyPair, err := signer.getSigningCertKeyPair() + if err != nil { + errs = append(errs, err) + } + if signingCertKeyPair == nil { + continue + } + signers = append(signers, signingCertKeyPair.Config.Certs[0]) + } + if len(errs) > 0 { + return errors.NewAggregate(errs) + } + if len(signers) == 0 { + return fmt.Errorf("No signers received yet") + } + _, err := c.CABundle.ensureConfigMapCABundleFromCerts(ctx, signers) + return err +} + +func (c RotatedCABundleController) Sync(ctx context.Context, syncCtx factory.SyncContext) error { + syncErr := c.SyncWorker(ctx, syncCtx) // running this function with RunOnceContextKey value context will make this "run-once" without updating status. isRunOnce, ok := ctx.Value(RunOnceContextKey).(bool) @@ -110,7 +150,7 @@ func (c CertRotationController) Sync(ctx context.Context, syncCtx factory.SyncCo return syncErr } - updated, updateErr := c.StatusReporter.Report(ctx, c.Name, syncErr) + updated, updateErr := c.StatusReporter.Report(ctx, c.name, syncErr) if updateErr != nil { return updateErr } @@ -121,27 +161,81 @@ func (c CertRotationController) Sync(ctx context.Context, syncCtx factory.SyncCo return syncErr } -func (c CertRotationController) SyncWorker(ctx context.Context) error { - signingCertKeyPair, _, err := c.RotatedSigningCASecret.EnsureSigningCertKeyPair(ctx) - if err != nil { - return err +// RotatedTargetSecretController continuously creates a target cert and key signed by the latest signing CA and store it in a secret +type RotatedTargetSecretController struct { + name string + + Target RotatedSelfSignedCertKeySecret + Signer *RotatedSigningCASecret + CABundle *CABundleConfigMap + // Plumbing: + StatusReporter StatusReporter +} + +func NewRotatedTargetSecretController( + target RotatedSelfSignedCertKeySecret, + signer *RotatedSigningCASecret, + cabundle *CABundleConfigMap, + recorder events.Recorder, + reporter StatusReporter, +) factory.Controller { + name := fmt.Sprintf("target %s/%s", target.Namespace, target.Name) + c := &RotatedTargetSecretController{ + Target: target, + Signer: signer, + CABundle: cabundle, + StatusReporter: reporter, + name: name, } + return factory.New(). + ResyncEvery(time.Minute). + WithSync(c.Sync). + WithInformers( + signer.Informer.Informer(), + cabundle.Informer.Informer(), + target.Informer.Informer(), + ). + WithPostStartHooks( + c.targetCertRecheckerPostRunHook, + ). + ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name)) +} - cabundleCerts, err := c.CABundleConfigMap.EnsureConfigMapCABundle(ctx, signingCertKeyPair) - if err != nil { +func (c RotatedTargetSecretController) SyncWorker(ctx context.Context, syncCtx factory.SyncContext) error { + signingCertKeyPair, err := c.Signer.getSigningCertKeyPair() + if err != nil || signingCertKeyPair == nil { return err } - - if _, err := c.RotatedSelfSignedCertKeySecret.EnsureTargetCertKeyPair(ctx, signingCertKeyPair, cabundleCerts); err != nil { + cabundleCerts, err := c.CABundle.getConfigMapCABundle() + if err != nil || cabundleCerts == nil { + return err + } + if _, err := c.Target.EnsureTargetCertKeyPair(ctx, signingCertKeyPair, cabundleCerts); err != nil { return err } - return nil } -func (c CertRotationController) targetCertRecheckerPostRunHook(ctx context.Context, syncCtx factory.SyncContext) error { +func (c RotatedTargetSecretController) Sync(ctx context.Context, syncCtx factory.SyncContext) error { + syncErr := c.SyncWorker(ctx, syncCtx) + + updated, updateErr := c.StatusReporter.Report(ctx, c.name, syncErr) + if updateErr != nil { + return updateErr + } + if updated && syncErr != nil { + syncCtx.Recorder().Warningf("RotationError", syncErr.Error()) + } + + return syncErr +} + +func (c RotatedTargetSecretController) targetCertRecheckerPostRunHook(ctx context.Context, syncCtx factory.SyncContext) error { + if c.Target.CertCreator == nil { + return nil + } // If we have a need to force rechecking the cert, use this channel to do it. - refresher, ok := c.RotatedSelfSignedCertKeySecret.CertCreator.(TargetCertRechecker) + refresher, ok := c.Target.CertCreator.(TargetCertRechecker) if !ok { return nil } diff --git a/pkg/operator/certrotation/client_cert_rotation_controllers_test.go b/pkg/operator/certrotation/client_cert_rotation_controllers_test.go new file mode 100644 index 0000000000..e87208a0b4 --- /dev/null +++ b/pkg/operator/certrotation/client_cert_rotation_controllers_test.go @@ -0,0 +1,652 @@ +package certrotation + +import ( + "context" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/events" + corev1 "k8s.io/api/core/v1" + "k8s.io/apiserver/pkg/authentication/user" + kubefake "k8s.io/client-go/kubernetes/fake" + corev1listers "k8s.io/client-go/listers/core/v1" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" +) + +type MockStatusReporter struct { +} + +func (s MockStatusReporter) Report(ctx context.Context, controllerName string, syncErr error) (bool, error) { + return false, nil +} + +func TestRotatedSigningCASecretController(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + + c := &RotatedSigningCASecretController{ + Signer: &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + }, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err := c.Sync(context.TODO(), factory.NewSyncContext("test", recorder)) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + if !actions[0].Matches("get", "secrets") { + t.Error(actions[0]) + } + if !actions[1].Matches("create", "secrets") { + t.Error(actions[1]) + } +} + +func TestRotatedSigningCASecretControllerParallel(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + + var workerWg sync.WaitGroup + nParallel := 4 + for range nParallel { + workerWg.Add(1) + go func() { + c := &RotatedSigningCASecretController{ + Signer: &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + }, + StatusReporter: MockStatusReporter{}, + name: "test", + } + c.Sync(context.TODO(), factory.NewSyncContext("test", recorder)) + workerWg.Done() + }() + } + + workerWg.Wait() + actions := client.Actions() + if len(actions) != nParallel*2 { + t.Fatal(spew.Sdump(actions)) + } + created := false + for i := 0; i < nParallel*2; i += 2 { + if !actions[i].Matches("get", "secrets") { + t.Error(actions[i]) + } + updateOrCreate := "create" + if created { + updateOrCreate = "update" + } else { + created = true + } + if !actions[i+1].Matches(updateOrCreate, "secrets") { + t.Error(actions[i+1]) + } + } +} + +func TestRotatedCABundleController(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + ctx := context.TODO() + syncCtx := factory.NewSyncContext("test", recorder) + + signer := &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + } + signerCtrl := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: MockStatusReporter{}, + name: "test", + } + + err := signerCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + signerSecret := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.Secret) + indexer.Add(signerSecret) + client.ClearActions() + + c := &RotatedCABundleController{ + CABundle: &CABundleConfigMap{ + Namespace: "ns", + Name: "cabundle", + Client: client.CoreV1(), + Lister: corev1listers.NewConfigMapLister(indexer), + EventRecorder: recorder, + }, + Signers: []*RotatedSigningCASecret{signer}, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err = c.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + if !actions[0].Matches("get", "configmaps") { + t.Error(actions[0]) + } + if !actions[1].Matches("create", "configmaps") { + t.Error(actions[1]) + } +} + +func TestRotatedCABundleControllerParallel(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + ctx := context.TODO() + syncCtx := factory.NewSyncContext("test", recorder) + + signer := &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + } + signerCtrl := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: MockStatusReporter{}, + name: "test", + } + + err := signerCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + signerSecret := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.Secret) + indexer.Add(signerSecret) + client.ClearActions() + + var workerWg sync.WaitGroup + nParallel := 4 + for range nParallel { + workerWg.Add(1) + go func() { + c := &RotatedCABundleController{ + CABundle: &CABundleConfigMap{ + Namespace: "ns", + Name: "cabundle", + Client: client.CoreV1(), + Lister: corev1listers.NewConfigMapLister(indexer), + EventRecorder: recorder, + }, + Signers: []*RotatedSigningCASecret{signer}, + StatusReporter: MockStatusReporter{}, + name: "test", + } + c.Sync(ctx, syncCtx) + workerWg.Done() + }() + } + workerWg.Wait() + + actions = client.Actions() + if len(actions) != nParallel+1 { + t.Fatal(spew.Sdump(actions)) + } + if !actions[0].Matches("get", "configmaps") { + t.Error(actions[0]) + } + if !actions[1].Matches("create", "configmaps") { + t.Error(actions[1]) + } + for i := 2; i < nParallel; i++ { + if !actions[i].Matches("get", "configmaps") { + t.Error(actions[i]) + } + } +} + +func TestRotatedCABundleMultipleSignersController(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + ctx := context.TODO() + syncCtx := factory.NewSyncContext("test", recorder) + + signers := []*RotatedSigningCASecret{} + signerCerts := []string{} + nSigners := 3 + for i := range nSigners { + signer := &RotatedSigningCASecret{ + Namespace: "ns", + Name: fmt.Sprintf("signer-%d-secret", i), + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + } + signers = append(signers, signer) + + signerCtrl := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: MockStatusReporter{}, + name: "test", + } + + err := signerCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + client.ClearActions() + signerSecret := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.Secret) + signerContents, ok := signerSecret.Data["tls.crt"] + if !ok { + t.Fatal(spew.Sdump(signerContents)) + } + signerCerts = append(signerCerts, string(signerContents)) + indexer.Add(signerSecret) + } + + c := &RotatedCABundleController{ + CABundle: &CABundleConfigMap{ + Namespace: "ns", + Name: "cabundle", + Client: client.CoreV1(), + Lister: corev1listers.NewConfigMapLister(indexer), + EventRecorder: recorder, + }, + Signers: signers, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err := c.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + if !actions[0].Matches("get", "configmaps") { + t.Error(actions[0]) + } + if !actions[1].Matches("create", "configmaps") { + t.Error(actions[1]) + } + caBundleConfigMap := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) + caBundleContents, ok := caBundleConfigMap.Data["ca-bundle.crt"] + if !ok { + t.Fatal(spew.Sdump(caBundleContents)) + } + for i := range nSigners { + signer := signerCerts[i] + if !strings.Contains(caBundleContents, signer) { + t.Fatalf("Missing signer #%d", i) + } + } + +} + +func TestRotatedTargetSecretController(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + ctx := context.TODO() + syncCtx := factory.NewSyncContext("test", recorder) + + signer := &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + } + signerCtrl := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: MockStatusReporter{}, + name: "test", + } + + err := signerCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + signerSecret := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.Secret) + indexer.Add(signerSecret) + client.ClearActions() + + caBundle := &CABundleConfigMap{ + Namespace: "ns", + Name: "cabundle", + Client: client.CoreV1(), + Lister: corev1listers.NewConfigMapLister(indexer), + EventRecorder: recorder, + } + caBundleCtrl := &RotatedCABundleController{ + CABundle: caBundle, + Signers: []*RotatedSigningCASecret{signer}, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err = caBundleCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + caBundleConfigMap := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) + indexer.Add(caBundleConfigMap) + client.ClearActions() + + targetCtrl := &RotatedTargetSecretController{ + CABundle: caBundle, + Signer: signer, + Target: RotatedSelfSignedCertKeySecret{ + Namespace: "ns", + Name: "target", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + CertCreator: &ClientRotation{ + UserInfo: &user.DefaultInfo{Name: "system:test"}, + }, + }, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err = targetCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + if !actions[0].Matches("get", "secrets") { + t.Error(actions[0]) + } + if !actions[1].Matches("create", "secrets") { + t.Error(actions[1]) + } +} + +func TestRotatedTargetSecretControllerParallel(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + ctx := context.TODO() + syncCtx := factory.NewSyncContext("test", recorder) + + signer := &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + } + signerCtrl := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: MockStatusReporter{}, + name: "test", + } + + err := signerCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + signerSecret := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.Secret) + indexer.Add(signerSecret) + client.ClearActions() + + caBundle := &CABundleConfigMap{ + Namespace: "ns", + Name: "cabundle", + Client: client.CoreV1(), + Lister: corev1listers.NewConfigMapLister(indexer), + EventRecorder: recorder, + } + caBundleCtrl := &RotatedCABundleController{ + CABundle: caBundle, + Signers: []*RotatedSigningCASecret{signer}, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err = caBundleCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + caBundleConfigMap := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) + indexer.Add(caBundleConfigMap) + client.ClearActions() + + var workerWg sync.WaitGroup + nParallel := 4 + for range nParallel { + workerWg.Add(1) + go func() { + c := &RotatedTargetSecretController{ + CABundle: caBundle, + Signer: signer, + Target: RotatedSelfSignedCertKeySecret{ + Namespace: "ns", + Name: "target", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + CertCreator: &ClientRotation{ + UserInfo: &user.DefaultInfo{Name: "system:test"}, + }, + }, + StatusReporter: MockStatusReporter{}, + name: "test", + } + c.Sync(ctx, syncCtx) + workerWg.Done() + }() + } + workerWg.Wait() + actions = client.Actions() + if len(actions) != nParallel*2 { + t.Fatal(spew.Sdump(actions)) + } + created := false + for i := 0; i < nParallel*2; i += 2 { + if !actions[i].Matches("get", "secrets") { + t.Error(actions[i]) + } + updateOrCreate := "create" + if created { + updateOrCreate = "update" + } else { + created = true + } + if !actions[i+1].Matches(updateOrCreate, "secrets") { + t.Error(actions[i+1]) + } + } +} + +func TestMultipleRotatedTargetSecretController(t *testing.T) { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + + client := kubefake.NewSimpleClientset() + recorder := events.NewInMemoryRecorder("test") + ctx := context.TODO() + syncCtx := factory.NewSyncContext("test", recorder) + + signer := &RotatedSigningCASecret{ + Namespace: "ns", + Name: "signer-secret", + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + } + signerCtrl := &RotatedSigningCASecretController{ + Signer: signer, + StatusReporter: MockStatusReporter{}, + name: "test", + } + + err := signerCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + signerSecret := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.Secret) + indexer.Add(signerSecret) + client.ClearActions() + + caBundle := &CABundleConfigMap{ + Namespace: "ns", + Name: "cabundle", + Client: client.CoreV1(), + Lister: corev1listers.NewConfigMapLister(indexer), + EventRecorder: recorder, + } + caBundleCtrl := &RotatedCABundleController{ + CABundle: caBundle, + Signers: []*RotatedSigningCASecret{signer}, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err = caBundleCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatal(spew.Sdump(actions)) + } + caBundleConfigMap := actions[1].(clienttesting.UpdateAction).GetObject().(*corev1.ConfigMap) + indexer.Add(caBundleConfigMap) + client.ClearActions() + + nTargets := 3 + for i := range nTargets { + targetCtrl := &RotatedTargetSecretController{ + CABundle: caBundle, + Signer: signer, + Target: RotatedSelfSignedCertKeySecret{ + Namespace: "ns", + Name: fmt.Sprintf("target-%d", i), + Validity: 24 * time.Hour, + Refresh: 12 * time.Hour, + Client: client.CoreV1(), + Lister: corev1listers.NewSecretLister(indexer), + EventRecorder: recorder, + UseSecretUpdateOnly: false, + CertCreator: &ClientRotation{ + UserInfo: &user.DefaultInfo{Name: "system:user-one"}, + }, + }, + StatusReporter: MockStatusReporter{}, + name: "test", + } + err = targetCtrl.Sync(ctx, syncCtx) + if err != nil { + t.Fatal(err) + } + } + + actions = client.Actions() + if len(actions) != nTargets*2 { + t.Fatal(spew.Sdump(actions)) + } + for i := 0; i < nTargets*2; i += 2 { + if !actions[i].Matches("get", "secrets") { + t.Error(actions[i]) + } + if !actions[i+1].Matches("create", "secrets") { + t.Error(actions[i+1]) + } + } +} diff --git a/pkg/operator/certrotation/signer.go b/pkg/operator/certrotation/signer.go index 36f3cf292d..68f4cbb5fb 100644 --- a/pkg/operator/certrotation/signer.go +++ b/pkg/operator/certrotation/signer.go @@ -123,6 +123,14 @@ func (c RotatedSigningCASecret) EnsureSigningCertKeyPair(ctx context.Context) (* return signingCertKeyPair, signerUpdated, nil } +func (c RotatedSigningCASecret) getSigningCertKeyPair() (*crypto.CA, error) { + signingCertKeyPairSecret, err := c.Lister.Secrets(c.Namespace).Get(c.Name) + if err != nil || apierrors.IsNotFound(err) || signingCertKeyPairSecret == nil { + return nil, err + } + return crypto.GetCAFromBytes(signingCertKeyPairSecret.Data["tls.crt"], signingCertKeyPairSecret.Data["tls.key"]) +} + // ensureOwnerReference adds the owner to the list of owner references in meta, if necessary func ensureOwnerReference(meta *metav1.ObjectMeta, owner *metav1.OwnerReference) bool { var found bool diff --git a/pkg/operator/certrotation/status_reporter.go b/pkg/operator/certrotation/status_reporter.go new file mode 100644 index 0000000000..15fc79ebea --- /dev/null +++ b/pkg/operator/certrotation/status_reporter.go @@ -0,0 +1,36 @@ +package certrotation + +import ( + "context" + "fmt" + + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/openshift/library-go/pkg/operator/condition" + "github.com/openshift/library-go/pkg/operator/v1helpers" +) + +// StatusReporter knows how to report the status of cert rotation +type StatusReporter interface { + Report(ctx context.Context, controllerName string, syncErr error) (updated bool, updateErr error) +} + +var _ StatusReporter = (*StaticPodConditionStatusReporter)(nil) + +type StaticPodConditionStatusReporter struct { + // Plumbing: + OperatorClient v1helpers.StaticPodOperatorClient +} + +func (s *StaticPodConditionStatusReporter) Report(ctx context.Context, controllerName string, syncErr error) (bool, error) { + newCondition := operatorv1.OperatorCondition{ + Type: fmt.Sprintf(condition.CertRotationDegradedConditionTypeFmt, controllerName), + Status: operatorv1.ConditionFalse, + } + if syncErr != nil { + newCondition.Status = operatorv1.ConditionTrue + newCondition.Reason = "RotationError" + newCondition.Message = syncErr.Error() + } + _, updated, updateErr := v1helpers.UpdateStaticPodStatus(ctx, s.OperatorClient, v1helpers.UpdateStaticPodConditionFn(newCondition)) + return updated, updateErr +}