Skip to content

Commit

Permalink
New cert rotation controlers
Browse files Browse the repository at this point in the history
  • Loading branch information
@vrutkovs
vrutkovs committed Jul 10, 2024
1 parent b941412 commit 8aaf652
Show file tree
Hide file tree
Showing 5 changed files with 883 additions and 72 deletions.
31 changes: 26 additions & 5 deletions pkg/operator/certrotation/cabundle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type CABundleConfigMap struct {
EventRecorder events.Recorder
}

func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingCertKeyPair *crypto.CA) ([]*x509.Certificate, error) {
func (c CABundleConfigMap) ensureConfigMapCABundleFromCerts(ctx context.Context, signers []*x509.Certificate) ([]*x509.Certificate, error) {
// by this point we have current signing cert/key pair. We now need to make sure that the ca-bundle configmap has this cert and
// doesn't have any expired certs
modified := false
Expand All @@ -68,7 +68,7 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
needsMetadataUpdate := c.AdditionalAnnotations.EnsureTLSMetadataUpdate(&caBundleConfigMap.ObjectMeta)
modified = needsOwnerUpdate || needsMetadataUpdate || modified

updatedCerts, err := manageCABundleConfigMap(caBundleConfigMap, signingCertKeyPair.Config.Certs[0])
updatedCerts, err := manageCABundleConfigMap(caBundleConfigMap, signers)
if err != nil {
return nil, err
}
Expand All @@ -86,7 +86,7 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
return nil, err
}
if updated {
klog.V(2).Infof("Updated ca-bundle.crt configmap %s/%s with:\n%s", certs.CertificateBundleToString(updatedCerts), caBundleConfigMap.Namespace, caBundleConfigMap.Name)
klog.V(2).Infof("Updated ca-bundle.crt configmap %s/%s with:\n%s", caBundleConfigMap.Namespace, caBundleConfigMap.Name, certs.CertificateBundleToString(updatedCerts))
}
caBundleConfigMap = actualCABundleConfigMap
}
Expand All @@ -103,9 +103,30 @@ func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingC
return certificates, nil
}

func (c CABundleConfigMap) EnsureConfigMapCABundle(ctx context.Context, signingCertKeyPair *crypto.CA) ([]*x509.Certificate, error) {
return c.ensureConfigMapCABundleFromCerts(ctx, signingCertKeyPair.Config.Certs)
}

func (c CABundleConfigMap) getConfigMapCABundle() ([]*x509.Certificate, error) {
caBundleConfigMap, err := c.Lister.ConfigMaps(c.Namespace).Get(c.Name)
if err != nil || apierrors.IsNotFound(err) || caBundleConfigMap == nil {
return nil, err
}
caBundle := caBundleConfigMap.Data["ca-bundle.crt"]
if len(caBundle) == 0 {
return nil, fmt.Errorf("configmap/%s -n%s missing ca-bundle.crt", caBundleConfigMap.Name, caBundleConfigMap.Namespace)
}
certificates, err := cert.ParseCertsPEM([]byte(caBundle))
if err != nil {
return nil, err
}

return certificates, nil
}

// manageCABundleConfigMap adds the new certificate to the list of cabundles, eliminates duplicates, and prunes the list of expired
// certs to trust as signers
func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, currentSigner *x509.Certificate) ([]*x509.Certificate, error) {
func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, signers []*x509.Certificate) ([]*x509.Certificate, error) {
if caBundleConfigMap.Data == nil {
caBundleConfigMap.Data = map[string]string{}
}
Expand All @@ -119,7 +140,7 @@ func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, currentSigner
return nil, err
}
}
certificates = append([]*x509.Certificate{currentSigner}, certificates...)
certificates = append(signers, certificates...)
certificates = crypto.FilterExpiredCerts(certificates...)

finalCertificates := []*x509.Certificate{}
Expand Down
228 changes: 161 additions & 67 deletions pkg/operator/certrotation/client_cert_rotation_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,15 @@ package certrotation

import (
"context"
"crypto/x509"
"fmt"
"time"

operatorv1 "github.com/openshift/api/operator/v1"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"

"github.com/openshift/library-go/pkg/controller/factory"
"github.com/openshift/library-go/pkg/operator/condition"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"
)

const (
Expand All @@ -27,90 +26,131 @@ const (
RunOnceContextKey = "cert-rotation-controller.openshift.io/run-once"
)

// StatusReporter knows how to report the status of cert rotation
type StatusReporter interface {
Report(ctx context.Context, controllerName string, syncErr error) (updated bool, updateErr error)
// RotatedSigningCASecretController continuously creates a self-signed signing CA (via RotatedSigningCASecret) and store it in a secret.
type RotatedSigningCASecretController struct {
name string

// Signer rotates a self-signed signing CA stored in a secret.
Signer *RotatedSigningCASecret
// Plumbing:
StatusReporter StatusReporter
}

var _ StatusReporter = (*StaticPodConditionStatusReporter)(nil)
func NewRotatedSigningCASecretController(
signer *RotatedSigningCASecret,
recorder events.Recorder,
reporter StatusReporter,
) factory.Controller {
name := fmt.Sprintf("signer %s/%s", signer.Namespace, signer.Name)
c := &RotatedSigningCASecretController{
Signer: signer,
StatusReporter: reporter,
name: name,
}
return factory.New().
ResyncEvery(time.Minute).
WithSync(c.Sync).
WithInformers(
signer.Informer.Informer(),
).
ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name))
}

type StaticPodConditionStatusReporter struct {
// Plumbing:
OperatorClient v1helpers.StaticPodOperatorClient
func (c RotatedSigningCASecretController) SyncWorker(ctx context.Context, syncCtx factory.SyncContext) error {
_, _, err := c.Signer.EnsureSigningCertKeyPair(ctx)
return err
}

func (s *StaticPodConditionStatusReporter) Report(ctx context.Context, controllerName string, syncErr error) (bool, error) {
newCondition := operatorv1.OperatorCondition{
Type: fmt.Sprintf(condition.CertRotationDegradedConditionTypeFmt, controllerName),
Status: operatorv1.ConditionFalse,
func (c RotatedSigningCASecretController) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
syncErr := c.SyncWorker(ctx, syncCtx)

// running this function with RunOnceContextKey value context will make this "run-once" without updating status.
isRunOnce, ok := ctx.Value(RunOnceContextKey).(bool)
if ok && isRunOnce {
return syncErr
}
if syncErr != nil {
newCondition.Status = operatorv1.ConditionTrue
newCondition.Reason = "RotationError"
newCondition.Message = syncErr.Error()

updated, updateErr := c.StatusReporter.Report(ctx, c.name, syncErr)
if updateErr != nil {
return updateErr
}
if updated && syncErr != nil {
syncCtx.Recorder().Warningf("RotationError", syncErr.Error())
}
_, updated, updateErr := v1helpers.UpdateStaticPodStatus(ctx, s.OperatorClient, v1helpers.UpdateStaticPodConditionFn(newCondition))
return updated, updateErr

return syncErr
}

// CertRotationController does:
//
// 1) continuously create a self-signed signing CA (via RotatedSigningCASecret) and store it in a secret.
// 2) maintain a CA bundle ConfigMap with all not yet expired CA certs.
// 3) continuously create a target cert and key signed by the latest signing CA and store it in a secret.
type CertRotationController struct {
// controller name
Name string
// RotatedSigningCASecret rotates a self-signed signing CA stored in a secret.
RotatedSigningCASecret RotatedSigningCASecret
// CABundleConfigMap maintains a CA bundle config map, by adding new CA certs coming from rotatedSigningCASecret, and by removing expired old ones.
CABundleConfigMap CABundleConfigMap
// RotatedSelfSignedCertKeySecret rotates a key and cert signed by a signing CA and stores it in a secret.
RotatedSelfSignedCertKeySecret RotatedSelfSignedCertKeySecret
// RotatedCABundleController maintains a CA bundle ConfigMap with all not yet expired CA certs.
type RotatedCABundleController struct {
name string

CABundle *CABundleConfigMap
Signers []*RotatedSigningCASecret
// Plumbing:
StatusReporter StatusReporter
}

func NewCertRotationController(
name string,
rotatedSigningCASecret RotatedSigningCASecret,
caBundleConfigMap CABundleConfigMap,
rotatedSelfSignedCertKeySecret RotatedSelfSignedCertKeySecret,
func NewRotatedCABundleConfigMapController(
cabundle *CABundleConfigMap,
signers []*RotatedSigningCASecret,
recorder events.Recorder,
reporter StatusReporter,
) factory.Controller {
c := &CertRotationController{
Name: name,
RotatedSigningCASecret: rotatedSigningCASecret,
CABundleConfigMap: caBundleConfigMap,
RotatedSelfSignedCertKeySecret: rotatedSelfSignedCertKeySecret,
StatusReporter: reporter,
name := fmt.Sprintf("cabundle %s/%s", cabundle.Namespace, cabundle.Name)
c := &RotatedCABundleController{
CABundle: cabundle,
Signers: signers,
StatusReporter: reporter,
name: name,
}
return factory.New().
ctrlFactory := factory.New().
ResyncEvery(time.Minute).
WithSync(c.Sync).
WithInformers(
rotatedSigningCASecret.Informer.Informer(),
caBundleConfigMap.Informer.Informer(),
rotatedSelfSignedCertKeySecret.Informer.Informer(),
).
WithPostStartHooks(
c.targetCertRecheckerPostRunHook,
).
cabundle.Informer.Informer(),
)
for _, signer := range signers {
ctrlFactory = ctrlFactory.WithInformers(signer.Informer.Informer())
}

return ctrlFactory.
ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name))
}

func (c CertRotationController) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
syncErr := c.SyncWorker(ctx)
func (c RotatedCABundleController) SyncWorker(ctx context.Context, syncCtx factory.SyncContext) error {
var errs []error
var signers []*x509.Certificate
for _, signer := range c.Signers {
signingCertKeyPair, err := signer.getSigningCertKeyPair()
if err != nil {
errs = append(errs, err)
}
if signingCertKeyPair == nil {
continue
}
signers = append(signers, signingCertKeyPair.Config.Certs[0])
}
if len(errs) > 0 {
return errors.NewAggregate(errs)
}
if len(signers) == 0 {
return fmt.Errorf("No signers received yet")
}
_, err := c.CABundle.ensureConfigMapCABundleFromCerts(ctx, signers)
return err
}

func (c RotatedCABundleController) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
syncErr := c.SyncWorker(ctx, syncCtx)

// running this function with RunOnceContextKey value context will make this "run-once" without updating status.
isRunOnce, ok := ctx.Value(RunOnceContextKey).(bool)
if ok && isRunOnce {
return syncErr
}

updated, updateErr := c.StatusReporter.Report(ctx, c.Name, syncErr)
updated, updateErr := c.StatusReporter.Report(ctx, c.name, syncErr)
if updateErr != nil {
return updateErr
}
Expand All @@ -121,27 +161,81 @@ func (c CertRotationController) Sync(ctx context.Context, syncCtx factory.SyncCo
return syncErr
}

func (c CertRotationController) SyncWorker(ctx context.Context) error {
signingCertKeyPair, _, err := c.RotatedSigningCASecret.EnsureSigningCertKeyPair(ctx)
if err != nil {
return err
// RotatedTargetSecretController continuously creates a target cert and key signed by the latest signing CA and store it in a secret
type RotatedTargetSecretController struct {
name string

Target RotatedSelfSignedCertKeySecret
Signer *RotatedSigningCASecret
CABundle *CABundleConfigMap
// Plumbing:
StatusReporter StatusReporter
}

func NewRotatedTargetSecretController(
target RotatedSelfSignedCertKeySecret,
signer *RotatedSigningCASecret,
cabundle *CABundleConfigMap,
recorder events.Recorder,
reporter StatusReporter,
) factory.Controller {
name := fmt.Sprintf("target %s/%s", target.Namespace, target.Name)
c := &RotatedTargetSecretController{
Target: target,
Signer: signer,
CABundle: cabundle,
StatusReporter: reporter,
name: name,
}
return factory.New().
ResyncEvery(time.Minute).
WithSync(c.Sync).
WithInformers(
signer.Informer.Informer(),
cabundle.Informer.Informer(),
target.Informer.Informer(),
).
WithPostStartHooks(
c.targetCertRecheckerPostRunHook,
).
ToController("CertRotationController", recorder.WithComponentSuffix("cert-rotation-controller").WithComponentSuffix(name))
}

cabundleCerts, err := c.CABundleConfigMap.EnsureConfigMapCABundle(ctx, signingCertKeyPair)
if err != nil {
func (c RotatedTargetSecretController) SyncWorker(ctx context.Context, syncCtx factory.SyncContext) error {
signingCertKeyPair, err := c.Signer.getSigningCertKeyPair()
if err != nil || signingCertKeyPair == nil {
return err
}

if _, err := c.RotatedSelfSignedCertKeySecret.EnsureTargetCertKeyPair(ctx, signingCertKeyPair, cabundleCerts); err != nil {
cabundleCerts, err := c.CABundle.getConfigMapCABundle()
if err != nil || cabundleCerts == nil {
return err
}
if _, err := c.Target.EnsureTargetCertKeyPair(ctx, signingCertKeyPair, cabundleCerts); err != nil {
return err
}

return nil
}

func (c CertRotationController) targetCertRecheckerPostRunHook(ctx context.Context, syncCtx factory.SyncContext) error {
func (c RotatedTargetSecretController) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
syncErr := c.SyncWorker(ctx, syncCtx)

updated, updateErr := c.StatusReporter.Report(ctx, c.name, syncErr)
if updateErr != nil {
return updateErr
}
if updated && syncErr != nil {
syncCtx.Recorder().Warningf("RotationError", syncErr.Error())
}

return syncErr
}

func (c RotatedTargetSecretController) targetCertRecheckerPostRunHook(ctx context.Context, syncCtx factory.SyncContext) error {
if c.Target.CertCreator == nil {
return nil
}
// If we have a need to force rechecking the cert, use this channel to do it.
refresher, ok := c.RotatedSelfSignedCertKeySecret.CertCreator.(TargetCertRechecker)
refresher, ok := c.Target.CertCreator.(TargetCertRechecker)
if !ok {
return nil
}
Expand Down
Loading

0 comments on commit 8aaf652

Please sign in to comment.