Watch cert files and reload on change

admission-webhook/cert_reloader.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"crypto/tls"
+	"sync"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/sirupsen/logrus"
+)
+
+type CertLoader interface {
+	CertPath() string
+	KeyPath() string
+	LoadCertificate() (*tls.Certificate, error)
+}
+
+type CertReloader struct {
+	sync.Mutex
+	certPath    string
+	keyPath     string
+	certificate *tls.Certificate
+}
+
+func NewCertReloader(certPath, keyPath string) *CertReloader {
+	return &CertReloader{
+		certPath: certPath,
+		keyPath:  keyPath,
+	}
+}
+
+func (cr *CertReloader) CertPath() string {
+	return cr.certPath
+}
+
+func (cr *CertReloader) KeyPath() string {
+	return cr.keyPath
+}
+
+// LoadCertificate loads or reloads the certificate from disk.
+func (cr *CertReloader) LoadCertificate() (*tls.Certificate, error) {
+	cr.Lock()
+	defer cr.Unlock()
+
+	cert, err := tls.LoadX509KeyPair(cr.certPath, cr.keyPath)
+	if err != nil {
+		return nil, err
+	}
+	cr.certificate = &cert
+	return cr.certificate, nil
+}
+
+// GetCertificateFunc returns a function that can be assigned to tls.Config.GetCertificate
+func (cr *CertReloader) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+	return func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		return cr.certificate, nil
+	}
+}
+
+func watchCertFiles(certLoader CertLoader) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		logrus.Errorf("error creating watcher: %v", err)
+	}
+
+	go func() {
+		defer watcher.Close()
+
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Rename == fsnotify.Rename {
+					logrus.Infof("detected change in certificate file: %v", event.Name)
+					_, err := certLoader.LoadCertificate()
+					if err != nil {
+						logrus.Errorf("error reloading certificate: %v", err)
+					} else {
+						logrus.Infof("successfully reloaded certificate")
+					}
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					logrus.Errorf("watcher error returned !ok: %v", err)
+					return
+				}
+				logrus.Errorf("watcher error: %v", err)
+			}
+		}
+	}()
+
+	err = watcher.Add(certLoader.CertPath())
+	if err != nil {
+		logrus.Fatalf("error watching certificate file: %v", err)
+	}
+	err = watcher.Add(certLoader.KeyPath())
+	if err != nil {
+		logrus.Fatalf("error watching key file: %v", err)
+	}
+}

admission-webhook/cert_reloader_test.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"crypto/tls"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestCertReloader tests the reloading functionality of the certificate.
+func TestCertReloader(t *testing.T) {
+	// Create temporary cert and key files
+	tmpCertFile, err := os.CreateTemp("", "cert*.pem")
+	if err != nil {
+		t.Fatalf("Failed to create temp cert file: %v", err)
+	}
+	defer os.Remove(tmpCertFile.Name()) // clean up
+
+	tmpKeyFile, err := os.CreateTemp("", "key*.pem")
+	if err != nil {
+		t.Fatalf("Failed to create temp key file: %v", err)
+	}
+	defer os.Remove(tmpKeyFile.Name()) // clean up
+
+	// Write initial cert and key to temp files
+	initialCertData, _ := os.ReadFile("testdata/cert.pem")
+	if err := os.WriteFile(tmpCertFile.Name(), initialCertData, 0644); err != nil {
+		t.Fatalf("Failed to write to temp cert file: %v", err)
+	}
+
+	initialKeyData, _ := os.ReadFile("testdata/key.pem")
+	if err := os.WriteFile(tmpKeyFile.Name(), initialKeyData, 0644); err != nil {
+		t.Fatalf("Failed to write to temp key file: %v", err)
+	}
+
+	// Setup CertReloader with temp files
+	certReloader := NewCertReloader(tmpCertFile.Name(), tmpKeyFile.Name())
+	_, err = certReloader.LoadCertificate()
+	if err != nil {
+		t.Fatalf("Failed to load initial certificate: %v", err)
+	}
+
+	// Mocking a certificate change by writing new data to the files
+	newCertData, _ := os.ReadFile("testdata/cert.pem")
+	if err := os.WriteFile(tmpCertFile.Name(), newCertData, 0644); err != nil {
+		t.Fatalf("Failed to write new data to cert file: %v", err)
+	}
+
+	// Simulate reloading
+	_, err = certReloader.LoadCertificate()
+	if err != nil {
+		t.Fatalf("Failed to reload certificate: %v", err)
+	}
+}
+
+type mockCertLoader struct {
+	certPath     string
+	keyPath      string
+	loadCertFunc func() (*tls.Certificate, error)
+}
+
+func (m *mockCertLoader) CertPath() string {
+	return m.certPath
+}
+
+func (m *mockCertLoader) KeyPath() string {
+	return m.keyPath
+}
+
+func (m *mockCertLoader) LoadCertificate() (*tls.Certificate, error) {
+	return m.loadCertFunc()
+}
+
+func TestWatchingCertFiles(t *testing.T) {
+	tmpCertFile, err := os.CreateTemp("", "cert*.pem")
+	if err != nil {
+		t.Fatalf("Failed to create temp cert file: %v", err)
+	}
+	defer os.Remove(tmpCertFile.Name())
+
+	tmpKeyFile, err := os.CreateTemp("", "key*.pem")
+	if err != nil {
+		t.Fatalf("Failed to create temp key file: %v", err)
+	}
+	defer os.Remove(tmpKeyFile.Name())
+
+	loadCertFuncChan := make(chan bool)
+
+	cl := &mockCertLoader{
+		certPath: tmpCertFile.Name(),
+		keyPath:  tmpKeyFile.Name(),
+		loadCertFunc: func() (*tls.Certificate, error) {
+			loadCertFuncChan <- true
+			return &tls.Certificate{}, nil
+		},
+	}
+
+	go func() {
+		defer close(loadCertFuncChan)
+
+		called := <-loadCertFuncChan
+		assert.True(t, called)
+	}()
+
+	watchCertFiles(cl)
+
+	newCertData, _ := os.ReadFile("testdata/cert.pem")
+	if err := os.WriteFile(tmpCertFile.Name(), newCertData, 0644); err != nil {
+		t.Fatalf("Failed to write new data to cert file: %v", err)
+	}
+
+	<-loadCertFuncChan
+}

admission-webhook/go.mod

@@ -3,6 +3,7 @@ module github.com/kubernetes-sigs/windows-gmsa/admission-webhook
 go 1.21
 
 require (
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4

admission-webhook/go.sum

@@ -4,6 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=

admission-webhook/integration_tests/integration_test.go

@@ -2,6 +2,7 @@ package integrationtests
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"html/template"
@@ -365,6 +366,42 @@ func TestDeployV1CredSpecGetAllVersions(t *testing.T) {
 	assert.Equal(t, v1alpha1CredSpec.Object["credSpec"], v1CredSpec.Object["credSpec"])
 }
 
+func TestPossibleToUpdatePodWithNewCert(t *testing.T) {
+	/** TODO:
+		 * - update the webhook pod to use the new flag
+	     * - make a request to create a pod to make sure it works (already done)
+	     * - get a blessed certificate from the API server
+		 *   (https://github.com/kubernetes-sigs/windows-gmsa/blob/141/admission-webhook/deploy/create-signed-cert.sh)
+	     * - update existing secret in place and wait for the pod to get new secrets which can take time
+		 *   (https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-files-from-a-pod) - similar to what you are doing here
+	     * - kubectl exec into the running pod to see that the secret changed
+		 *   (using utils like https://github.com/ycheng-kareo/windows-gmsa/blob/watch-reload-cert/admission-webhook/integration_tests/kube.go#L199)
+	     * - make a request to create a pod to verify that it still works (pod := waitForPodToComeUp(t, testConfig.Namespace, "app="+testName))
+		 * - add a separate test to verify that requests to the webhook always return during this process
+	*/
+	testName := "possible-to-update-pod-with-new-cert"
+	credSpecTemplates := []string{"credspec-0"}
+	newSecretTemplate := "new-secret"
+	templates := []string{"credspecs-users-rbac-role", "service-account", "sa-rbac-binding", "single-pod-with-container-level-gmsa"}
+
+	testConfig, tearDownFunc := integrationTestSetup(t, testName, credSpecTemplates, templates)
+	defer tearDownFunc()
+
+	pod := waitForPodToComeUp(t, testConfig.Namespace, "app="+testName)
+	assert.Equal(t, expectedCredSpec0, extractContainerCredSpecContents(t, pod, testName))
+
+	// read test cert & key
+	newCert, _ := os.ReadFile("../testdata/cert.pem")
+	newKey, _ := os.ReadFile("../testdata/key.pem")
+	testConfig.Cert = base64.StdEncoding.EncodeToString(newCert)
+	testConfig.Key = base64.StdEncoding.EncodeToString(newKey)
+
+	// apply the new cert & key pair
+	renderedTemplate := renderTemplate(t, testConfig, newSecretTemplate)
+	success, _, _ := applyManifest(t, renderedTemplate)
+	assert.True(t, success)
+}
+
 /* Helpers */
 
 type testConfig struct {
@@ -378,6 +415,8 @@ type testConfig struct {
 	RoleBindingName    string
 	Image              string
 	ExtraSpecLines     []string
+	Cert               string
+	Key                string
 }
 
 // integrationTestSetup creates a new namespace to play in, and returns a function to

admission-webhook/integration_tests/templates/new-secret.yml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .TestName }}
+  namespace: {{ .Namespace }}
+data:
+  tls_private_key: {{ .Key }}
+  tls_certificate: {{ .Cert }}

admission-webhook/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"os"
 	"strconv"
@@ -13,12 +14,15 @@ import (
 func main() {
 	initLogrus()
 
+	enableCertReload := flag.Bool("cert-reload", false, "enable certificate reload")
+	flag.Parse()
+
 	kubeClient, err := createKubeClient()
 	if err != nil {
 		panic(err)
 	}
 
-	webhook := newWebhook(kubeClient)
+	webhook := newWebhookWithOptions(kubeClient, WithCertReload(*enableCertReload))
 
 	tlsConfig := &tlsConfig{
 		crtPath: env("TLS_CRT"),

admission-webhook/testdata/cert.pem

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCTCCAvGgAwIBAgIUDlTAXZUZ0DGcutGxcspszdn4lAowDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDIwODA2MDAyN1oXDTM0MDIw
+NTA2MDAyN1owFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAg1kVSkS0tl8+VMBH8nVwfXy3yNgDS5a8uWAFv62O1vI4
+KEOClNhPaFvihL7ubi3rhCmHoUfbIhe1MMqwV7mEXI+AVObs7Sf8feX5WkW6zJyF
+AByH8O37fGFfEsISJBcMwmS5/gCShPfUuG2vr7Zg28brxU+Ixp1DhA87X+A+CEG0
+JTH2LDiKMv86At/olH/IeDOH7j2tD25MThDN+xyKa9u2cpy73GcF822haUtFzmVX
+mA8Mw4Qs5B2lMPY3k/C2UaqRDnNFu+0U011hvAGFA4+Jw4Cvpy13/4kQQZ0JHSOD
+oy+jtbpqMQJn2oCMQ9DX0WQTS7E4W03y5gKS4v8xkUneAWuWoSwTm8TXRoAXbT3n
+ZqDXmdy69ckLiLgn/w5uAeKjeHdk522QiJ2MHqYRLJbzUQ6LsrYdcR3nhh1pgh5K
+tdnuz7HQtg77KR9g1X1aAT20SqV9rV85FwWI50dTfg8ehWXOSuXlZMlRUuMOn0a0
+iAyw+rCbaLvmuXwPZxuk/PPW+4lWwE1OqHSjs3iHl/fZM5AfmVS9Av3n3JRD2hrA
+2aoOnjiFSjI/9qzcjUl8LTvrzGFt3QWcVRDzMqNQW8qPvBFvxrRcZlPTElRM023p
+TO8P3k4n08EGgY+dV9s3xC6dnIkyVp7b2UtEDAC6E53mI2e+wG5uYrKu4QSaM9MC
+AwEAAaNTMFEwHQYDVR0OBBYEFCEw8jWYUa8ed+R5O9dxhUSVqBJwMB8GA1UdIwQY
+MBaAFCEw8jWYUa8ed+R5O9dxhUSVqBJwMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggIBAAE7F7qZ4B5PzqydYRJ0Er39kdfEugs0L3/LYwHfQ4eJKedI
+CNXnSW+2kh5kQW9YnXy77VewZ/wtaZyGndNNocRKlN6X2yr97HOtuysdSvHuNmUl
+Dyk9brgPcRJK4YizO44DHuQmn0LUhxbeph2VjYxs6B5XEdD6aGFpljWNCHzuWqao
+so3uF588lhudSPVkx9VEWHF/N5BKQeJr6gPy1BB8rlSkD8ImxHmq7ledV7ri0mCS
+o5WO+17kaLBvyj5H8sN/M+zWPKMHLohe5BXFWwlgl8nGnaXNW0HaKhgyjGTZBZJe
+u5kTVQnhTDUo706K4BC8Zz78L6Xcb44FMUIRF5yZ8iKN2M6mPmEQmrE6aKEmiNxc
+j0Yfz5MGumog8goYEgOkxp49aI6zojOq7GskDVKo9NxPsfotASriDOhpe106F/yK
+cboL1oeL3pAjgICgwdy2pNawjDVNt8cadU4RAxF+m0gpa/xAsGhlz5YKsdOv/7V8
+Lb7KguUyYHmyBFwzJluhBrUWrGpPEKxdjrMbn/9G8b7AbXyV2w/9bwqkLvFU6qyJ
+vuv4HpHIywsm9tST8p62RDVRbWlYfBwWIsnz4sraOPJXt8SU9QE3XI3MLw3iiEbs
+1oToxKEj+6KbAgaC81cIkohZ+6RYnX+huhhe89Mgg0r7wWnt+huHiKLhGnm/
+-----END CERTIFICATE-----