Certificate Rotation in APIServer (#1154)

Self signed certificates are now stored in files rather than memory. This allows for the use of dynamic certificate objects. That will automatically process the update

The caCertController is updated immediately through RunOnce, whereas the DynamicServingContent for the serving certificate can take up to one minute

Fixes #1152

pkg/apiserver/certificate/cacert_controller.go

@@ -89,22 +89,20 @@ func newCACertController(caContentProvider dynamiccertificates.CAContentProvider
 	return c
 }
 
-// UpdateCertificate updates the certificate to a new one. Used to rotate statically signed certificates before they
-// expire.
-func (c *CACertController) UpdateCertificate(caContentProvider dynamiccertificates.CAContentProvider) {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
-
-	c.caContentProvider = caContentProvider
+func (c *CACertController) UpdateCertificate() error {
+	if controller, ok := c.caContentProvider.(dynamiccertificates.ControllerRunner); ok {
+		if err := controller.RunOnce(); err != nil {
+			klog.Warningf("Updating of CA content failed: %v", err)
+			c.Enqueue()
+			return err
+		}
+	}
 
-	// Need to trigger a sync.
-	c.Enqueue()
+	return nil
 }
 
 // getCertificate exposes the certificate for testing.
 func (c *CACertController) getCertificate() []byte {
-	c.mutex.RLock()
-	defer c.mutex.RUnlock()
 	return c.caContentProvider.CurrentCABundleContent()
 }
 
@@ -115,11 +113,7 @@ func (c *CACertController) Enqueue() {
 }
 
 func (c *CACertController) syncCACert() error {
-	caCert := func() []byte {
-		c.mutex.RLock()
-		defer c.mutex.RUnlock()
-		return c.caContentProvider.CurrentCABundleContent()
-	}()
+	caCert := c.caContentProvider.CurrentCABundleContent()
 
 	if err := c.syncConfigMap(caCert); err != nil {
 		return err

pkg/apiserver/certificate/certificate.go

@@ -15,8 +15,6 @@
 package certificate
 
 import (
-	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"net"
 	"os"
@@ -27,6 +25,8 @@ import (
 	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/kubernetes"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
 	"k8s.io/klog"
 	"k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
@@ -36,6 +36,9 @@ import (
 var (
 	// certDir is the directory that the TLS Secret should be mounted to. Declaring it as a variable for testing.
 	certDir = "/var/run/antrea/antrea-controller-tls"
+
+	// selfSignedCertDir is the dir Antrea self signed certificates are created in.
+	selfSignedCertDir = "/var/run/antrea/antrea-controller-self-signed"
 	// certReadyTimeout is the timeout we will wait for the TLS Secret being ready. Declaring it as a variable for testing.
 	certReadyTimeout = 2 * time.Minute
 
@@ -123,16 +126,15 @@ func generateSelfSignedCertificate(secureServing *options.SecureServingOptionsWi
 	var err error
 	var caContentProvider dynamiccertificates.CAContentProvider
 
-	// Set the PairName but leave certificate directory blank to generate in-memory by default.
-	secureServing.ServerCert.CertDirectory = ""
+	// Set the PairName and CertDirectory to generate the certificate files.
+	secureServing.ServerCert.CertDirectory = selfSignedCertDir
 	secureServing.ServerCert.PairName = "antrea-controller"
 
 	if err := secureServing.MaybeDefaultWithSelfSignedCerts("antrea", GetAntreaServerNames(), []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
 		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
-	certPEMBlock, _ := secureServing.ServerCert.GeneratedCert.CurrentCertKeyContent()
-	caContentProvider, err = dynamiccertificates.NewStaticCAContent("self-signed cert", certPEMBlock)
+	caContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("self-signed cert", secureServing.ServerCert.CertKey.CertFile)
 	if err != nil {
 		return nil, fmt.Errorf("error reading self-signed CA certificate: %v", err)
 	}
@@ -145,20 +147,15 @@ func generateSelfSignedCertificate(secureServing *options.SecureServingOptionsWi
 // Also can be used to pass a user provided rotation window.
 func nextRotationDuration(secureServing *options.SecureServingOptionsWithLoopback,
 	maxRotateDuration time.Duration) (time.Duration, error) {
-	pemCert, pemKey := secureServing.ServerCert.GeneratedCert.CurrentCertKeyContent()
-	cert, err := tls.X509KeyPair(pemCert, pemKey)
-	if err != nil {
-		return time.Duration(0), fmt.Errorf("error parsing generated certificate: %v", err)
-	}
 
-	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+	x509Cert, err := certutil.CertsFromFile(secureServing.ServerCert.CertKey.CertFile)
 	if err != nil {
 		return time.Duration(0), fmt.Errorf("error parsing generated certificate: %v", err)
 	}
 
 	// Attempt to rotate the certificate at the half-way point of expiration.
 	// Unless the halfway point is longer than maxRotateDuration
-	duration := x509Cert.NotAfter.Sub(time.Now()) / 2
+	duration := x509Cert[0].NotAfter.Sub(time.Now()) / 2
 
 	waitDuration := duration
 	if maxRotateDuration < waitDuration {
@@ -178,15 +175,35 @@ func rotateSelfSignedCertificates(c *CACertController, secureServing *options.Se
 			klog.Errorf("error reading expiration date of cert: %v", err)
 			return
 		}
+
+		klog.Infof("Certificate will be rotated at %v", time.Now().Add(rotationDuration))
+
 		time.Sleep(rotationDuration)
 
 		klog.Infof("Rotating self signed certificate")
 
-		caContentProvider, err := generateSelfSignedCertificate(secureServing)
+		err = generateNewServingCertificate(secureServing)
 		if err != nil {
 			klog.Errorf("error generating new cert: %v", err)
 			return
 		}
-		c.UpdateCertificate(caContentProvider)
+		c.UpdateCertificate()
+	}
+}
+
+func generateNewServingCertificate(secureServing *options.SecureServingOptionsWithLoopback) error {
+	cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures("antrea", []net.IP{net.ParseIP("127.0.0.1")}, GetAntreaServerNames(), secureServing.ServerCert.FixtureDirectory)
+	if err != nil {
+		return fmt.Errorf("unable to generate self signed cert: %v", err)
 	}
+
+	if err := certutil.WriteCert(secureServing.ServerCert.CertKey.CertFile, cert); err != nil {
+		return err
+	}
+	if err := keyutil.WriteKey(secureServing.ServerCert.CertKey.KeyFile, key); err != nil {
+		return err
+	}
+	klog.Infof("Generated self-signed cert (%s, %s)", secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile)
+
+	return nil
 }

pkg/apiserver/certificate/certificate_test.go

@@ -157,11 +157,16 @@ func TestApplyServerCert(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			var err error
 			certDir, err = ioutil.TempDir("", "antrea-tls-test")
-			certReadyTimeout = 100 * time.Millisecond
 			if err != nil {
 				t.Fatalf("Unable to create temporary directory: %v", err)
 			}
 			defer os.RemoveAll(certDir)
+			selfSignedCertDir, err = ioutil.TempDir("", "antrea-self-signed")
+			if err != nil {
+				t.Fatalf("Unable to create temporary directory: %v", err)
+			}
+			defer os.RemoveAll(selfSignedCertDir)
+			certReadyTimeout = 100 * time.Millisecond
 			secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
 			if tt.tlsCert != nil {
 				certutil.WriteCert(path.Join(certDir, TLSCertFile), tt.tlsCert)
@@ -203,9 +208,9 @@ func TestApplyServerCert(t *testing.T) {
 				assert.Equal(t, genericoptions.CertKey{CertFile: certDir + "/tls.crt", KeyFile: certDir + "/tls.key"}, secureServing.ServerCert.CertKey, "CertKey doesn't match")
 			}
 			if tt.wantGeneratedCert {
-				assert.NotNil(t, secureServing.ServerCert.GeneratedCert)
+				assert.Equal(t, genericoptions.CertKey{CertFile: selfSignedCertDir + "/antrea-controller.crt", KeyFile: selfSignedCertDir + "/antrea-controller.key"}, secureServing.ServerCert.CertKey, "SelfSigned certs not generated")
 			} else {
-				assert.Nil(t, secureServing.ServerCert.GeneratedCert)
+				assert.NotEqual(t, genericoptions.CertKey{CertFile: selfSignedCertDir + "/antrea-controller.crt", KeyFile: selfSignedCertDir + "/antrea-controller.key"}, secureServing.ServerCert.CertKey, "SelfSigned certs generated erroneously")
 			}
 			if tt.wantCACert != nil {
 				assert.Equal(t, tt.wantCACert, got.caContentProvider.CurrentCABundleContent(), "CA cert doesn't match")