From 6bab6d65c5729630fb5d56ff28310f6fe07d4e0a Mon Sep 17 00:00:00 2001 From: MatthewHinton56 Date: Thu, 27 Aug 2020 15:20:26 -0500 Subject: [PATCH] Certificate Rotation in APIServer (#1154) Self signed certificates are now stored in files rather than memory. This allows for the use of dynamic certificate objects. That will automatically process the update The caCertController is updated immediately through RunOnce, whereas the DynamicServingContent for the serving certificate can take up to one minute Fixes #1152 --- .../certificate/cacert_controller.go | 26 ++++------ pkg/apiserver/certificate/certificate.go | 47 +++++++++++++------ pkg/apiserver/certificate/certificate_test.go | 11 +++-- 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/pkg/apiserver/certificate/cacert_controller.go b/pkg/apiserver/certificate/cacert_controller.go index 02cbdccca9c..6520558cac3 100644 --- a/pkg/apiserver/certificate/cacert_controller.go +++ b/pkg/apiserver/certificate/cacert_controller.go @@ -89,22 +89,20 @@ func newCACertController(caContentProvider dynamiccertificates.CAContentProvider return c } -// UpdateCertificate updates the certificate to a new one. Used to rotate statically signed certificates before they -// expire. -func (c *CACertController) UpdateCertificate(caContentProvider dynamiccertificates.CAContentProvider) { - c.mutex.Lock() - defer c.mutex.Unlock() - - c.caContentProvider = caContentProvider +func (c *CACertController) UpdateCertificate() error { + if controller, ok := c.caContentProvider.(dynamiccertificates.ControllerRunner); ok { + if err := controller.RunOnce(); err != nil { + klog.Warningf("Updating of CA content failed: %v", err) + c.Enqueue() + return err + } + } - // Need to trigger a sync. - c.Enqueue() + return nil } // getCertificate exposes the certificate for testing. func (c *CACertController) getCertificate() []byte { - c.mutex.RLock() - defer c.mutex.RUnlock() return c.caContentProvider.CurrentCABundleContent() } @@ -115,11 +113,7 @@ func (c *CACertController) Enqueue() { } func (c *CACertController) syncCACert() error { - caCert := func() []byte { - c.mutex.RLock() - defer c.mutex.RUnlock() - return c.caContentProvider.CurrentCABundleContent() - }() + caCert := c.caContentProvider.CurrentCABundleContent() if err := c.syncConfigMap(caCert); err != nil { return err diff --git a/pkg/apiserver/certificate/certificate.go b/pkg/apiserver/certificate/certificate.go index c0ed0e0f5a5..e2aa7a5dc65 100644 --- a/pkg/apiserver/certificate/certificate.go +++ b/pkg/apiserver/certificate/certificate.go @@ -15,8 +15,6 @@ package certificate import ( - "crypto/tls" - "crypto/x509" "fmt" "net" "os" @@ -27,6 +25,8 @@ import ( "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/kubernetes" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" "k8s.io/klog" "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" @@ -36,6 +36,9 @@ import ( var ( // certDir is the directory that the TLS Secret should be mounted to. Declaring it as a variable for testing. certDir = "/var/run/antrea/antrea-controller-tls" + + // selfSignedCertDir is the dir Antrea self signed certificates are created in. + selfSignedCertDir = "/var/run/antrea/antrea-controller-self-signed" // certReadyTimeout is the timeout we will wait for the TLS Secret being ready. Declaring it as a variable for testing. certReadyTimeout = 2 * time.Minute @@ -123,16 +126,15 @@ func generateSelfSignedCertificate(secureServing *options.SecureServingOptionsWi var err error var caContentProvider dynamiccertificates.CAContentProvider - // Set the PairName but leave certificate directory blank to generate in-memory by default. - secureServing.ServerCert.CertDirectory = "" + // Set the PairName and CertDirectory to generate the certificate files. + secureServing.ServerCert.CertDirectory = selfSignedCertDir secureServing.ServerCert.PairName = "antrea-controller" if err := secureServing.MaybeDefaultWithSelfSignedCerts("antrea", GetAntreaServerNames(), []net.IP{net.ParseIP("127.0.0.1")}); err != nil { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } - certPEMBlock, _ := secureServing.ServerCert.GeneratedCert.CurrentCertKeyContent() - caContentProvider, err = dynamiccertificates.NewStaticCAContent("self-signed cert", certPEMBlock) + caContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("self-signed cert", secureServing.ServerCert.CertKey.CertFile) if err != nil { return nil, fmt.Errorf("error reading self-signed CA certificate: %v", err) } @@ -145,20 +147,15 @@ func generateSelfSignedCertificate(secureServing *options.SecureServingOptionsWi // Also can be used to pass a user provided rotation window. func nextRotationDuration(secureServing *options.SecureServingOptionsWithLoopback, maxRotateDuration time.Duration) (time.Duration, error) { - pemCert, pemKey := secureServing.ServerCert.GeneratedCert.CurrentCertKeyContent() - cert, err := tls.X509KeyPair(pemCert, pemKey) - if err != nil { - return time.Duration(0), fmt.Errorf("error parsing generated certificate: %v", err) - } - x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + x509Cert, err := certutil.CertsFromFile(secureServing.ServerCert.CertKey.CertFile) if err != nil { return time.Duration(0), fmt.Errorf("error parsing generated certificate: %v", err) } // Attempt to rotate the certificate at the half-way point of expiration. // Unless the halfway point is longer than maxRotateDuration - duration := x509Cert.NotAfter.Sub(time.Now()) / 2 + duration := x509Cert[0].NotAfter.Sub(time.Now()) / 2 waitDuration := duration if maxRotateDuration < waitDuration { @@ -178,15 +175,35 @@ func rotateSelfSignedCertificates(c *CACertController, secureServing *options.Se klog.Errorf("error reading expiration date of cert: %v", err) return } + + klog.Infof("Certificate will be rotated at %v", time.Now().Add(rotationDuration)) + time.Sleep(rotationDuration) klog.Infof("Rotating self signed certificate") - caContentProvider, err := generateSelfSignedCertificate(secureServing) + err = generateNewServingCertificate(secureServing) if err != nil { klog.Errorf("error generating new cert: %v", err) return } - c.UpdateCertificate(caContentProvider) + c.UpdateCertificate() + } +} + +func generateNewServingCertificate(secureServing *options.SecureServingOptionsWithLoopback) error { + cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures("antrea", []net.IP{net.ParseIP("127.0.0.1")}, GetAntreaServerNames(), secureServing.ServerCert.FixtureDirectory) + if err != nil { + return fmt.Errorf("unable to generate self signed cert: %v", err) } + + if err := certutil.WriteCert(secureServing.ServerCert.CertKey.CertFile, cert); err != nil { + return err + } + if err := keyutil.WriteKey(secureServing.ServerCert.CertKey.KeyFile, key); err != nil { + return err + } + klog.Infof("Generated self-signed cert (%s, %s)", secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile) + + return nil } diff --git a/pkg/apiserver/certificate/certificate_test.go b/pkg/apiserver/certificate/certificate_test.go index f590579a289..f8cfa898fd2 100644 --- a/pkg/apiserver/certificate/certificate_test.go +++ b/pkg/apiserver/certificate/certificate_test.go @@ -157,11 +157,16 @@ func TestApplyServerCert(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var err error certDir, err = ioutil.TempDir("", "antrea-tls-test") - certReadyTimeout = 100 * time.Millisecond if err != nil { t.Fatalf("Unable to create temporary directory: %v", err) } defer os.RemoveAll(certDir) + selfSignedCertDir, err = ioutil.TempDir("", "antrea-self-signed") + if err != nil { + t.Fatalf("Unable to create temporary directory: %v", err) + } + defer os.RemoveAll(selfSignedCertDir) + certReadyTimeout = 100 * time.Millisecond secureServing := genericoptions.NewSecureServingOptions().WithLoopback() if tt.tlsCert != nil { certutil.WriteCert(path.Join(certDir, TLSCertFile), tt.tlsCert) @@ -203,9 +208,9 @@ func TestApplyServerCert(t *testing.T) { assert.Equal(t, genericoptions.CertKey{CertFile: certDir + "/tls.crt", KeyFile: certDir + "/tls.key"}, secureServing.ServerCert.CertKey, "CertKey doesn't match") } if tt.wantGeneratedCert { - assert.NotNil(t, secureServing.ServerCert.GeneratedCert) + assert.Equal(t, genericoptions.CertKey{CertFile: selfSignedCertDir + "/antrea-controller.crt", KeyFile: selfSignedCertDir + "/antrea-controller.key"}, secureServing.ServerCert.CertKey, "SelfSigned certs not generated") } else { - assert.Nil(t, secureServing.ServerCert.GeneratedCert) + assert.NotEqual(t, genericoptions.CertKey{CertFile: selfSignedCertDir + "/antrea-controller.crt", KeyFile: selfSignedCertDir + "/antrea-controller.key"}, secureServing.ServerCert.CertKey, "SelfSigned certs generated erroneously") } if tt.wantCACert != nil { assert.Equal(t, tt.wantCACert, got.caContentProvider.CurrentCABundleContent(), "CA cert doesn't match")