Generate user certificates on the fly in tests

This way we can create rest clients with authentication details that
match the needs of the test instead of reusing credentials that have
been set by the CI. This simplifies setup significantly.

Co-authored-by: Danail Branekov <danailster@gmail.com>
Co-authored-by: Georgi Sabev <georgethebeatle@gmail.com>

api/authorization/user_client_factory_test.go

@@ -237,7 +237,7 @@ var _ = Describe("Unprivileged User Client Factory", func() {
 
 		When("the cert is not valid on this cluster", func() {
 			BeforeEach(func() {
-				authInfo.CertData = helpers.CreateCertificatePEM()
+				authInfo.CertData = helpers.CreateSelfSignedCertificatePEM()
 			})
 
 			It("creates an unusable client", func() {

tests/e2e/authorization_test.go

@@ -1,7 +1,6 @@
 package e2e_test
 
 import (
-	"crypto/tls"
 	"net/http"
 
 	"code.cloudfoundry.org/korifi/tests/helpers"
@@ -13,9 +12,6 @@ import (
 
 var _ = Describe("Authorization", func() {
 	var (
-		serviceaccountFactory *helpers.ServiceAccountFactory
-		svcAcctName           string
-
 		userName   string
 		userClient *helpers.CorrelatedRestyClient
 
@@ -24,18 +20,12 @@ var _ = Describe("Authorization", func() {
 	)
 
 	BeforeEach(func() {
-		serviceaccountFactory = helpers.NewServiceAccountFactory(rootNamespace)
-		svcAcctName = uuid.NewString()
-
-		userName = "system:serviceaccount:cf:" + svcAcctName
-		userToken := serviceaccountFactory.CreateServiceAccount(svcAcctName)
-		userClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
-			SetAuthToken(userToken).
-			SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+		userName = uuid.NewString()
+		userClient = makeTokenClient(serviceAccountFactory.CreateServiceAccount(userName))
 	})
 
 	AfterEach(func() {
-		serviceaccountFactory.DeleteServiceAccount(svcAcctName)
+		serviceAccountFactory.DeleteServiceAccount(userName)
 	})
 
 	Describe("Unauthorized users", func() {
@@ -77,11 +67,11 @@ var _ = Describe("Authorization", func() {
 		BeforeEach(func() {
 			orgName := generateGUID("org")
 			orgGUID = createOrg(orgName)
-			createOrgRole("organization_user", userName, orgGUID)
+			createOrgRole("organization_user", serviceAccountFactory.FullyQualifiedName(userName), orgGUID)
 
 			spaceName := generateGUID("space")
 			spaceGUID = createSpace(spaceName, orgGUID)
-			createSpaceRole("space_developer", userName, spaceGUID)
+			createSpaceRole("space_developer", serviceAccountFactory.FullyQualifiedName(userName), spaceGUID)
 		})
 
 		AfterEach(func() {

tests/e2e/e2e_suite_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -35,24 +36,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
+const adminUserName = "cf-admin"
+
 var (
 	correlationId string
 
-	adminClient                    *helpers.CorrelatedRestyClient
-	privilegedServiceAccountClient *helpers.CorrelatedRestyClient
-	longCertClient                 *helpers.CorrelatedRestyClient
+	serviceAccountFactory *helpers.ServiceAccountFactory
+	adminServiceAccount   string
+	adminClient           *helpers.CorrelatedRestyClient
 
 	apiServerRoot string
 
-	serviceAccountName  string
-	serviceAccountToken string
-
-	certUserName string
-	certPEM      string
-
-	longCertUserName string
-	longCertPEM      string
-
 	rootNamespace           string
 	appFQDN                 string
 	commonTestOrgGUID       string
@@ -307,14 +301,22 @@ func TestE2E(t *testing.T) {
 }
 
 type sharedSetupData struct {
-	CommonOrgName           string `json:"commonOrgName"`
-	CommonOrgGUID           string `json:"commonOrgGuid"`
-	DefaultAppBitsFile      string `json:"defaultAppBitsFile"`
-	MultiProcessAppBitsFile string `json:"multiProcessAppBitsFile"`
+	CommonOrgName            string `json:"commonOrgName"`
+	CommonOrgGUID            string `json:"commonOrgGuid"`
+	DefaultAppBitsFile       string `json:"defaultAppBitsFile"`
+	MultiProcessAppBitsFile  string `json:"multiProcessAppBitsFile"`
+	AdminServiceAccount      string `json:"admin_service_account"`
+	AdminServiceAccountToken string `json:"admin_service_account_token"`
 }
 
 var _ = SynchronizedBeforeSuite(func() []byte {
 	commonTestSetup()
+
+	adminServiceAccount = uuid.NewString()
+	adminServiceAccountToken := serviceAccountFactory.CreateAdminServiceAccount(adminServiceAccount)
+
+	adminClient = makeTokenClient(adminServiceAccountToken)
+
 	commonTestOrgName = generateGUID("common-test-org")
 	commonTestOrgGUID = createOrg(commonTestOrgName)
 
@@ -329,15 +331,19 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		// The DEFAULT_APP_BITS_PATH and DEFAULT_APP_RESPONSE environment variables are a workaround to allow e2e tests to run
 		// with a different app in these environments.
 		// See https://github.com/cloudfoundry/korifi/issues/2355 for refactoring ideas
-		DefaultAppBitsFile:      zipAsset(helpers.GetDefaultedEnvVar("DEFAULT_APP_BITS_PATH", "../assets/dorifi")),
-		MultiProcessAppBitsFile: zipAsset("../assets/multi-process"),
+		DefaultAppBitsFile:       zipAsset(helpers.GetDefaultedEnvVar("DEFAULT_APP_BITS_PATH", "../assets/dorifi")),
+		MultiProcessAppBitsFile:  zipAsset("../assets/multi-process"),
+		AdminServiceAccount:      adminServiceAccount,
+		AdminServiceAccountToken: adminServiceAccountToken,
 	}
 
 	bs, err := json.Marshal(sharedData)
 	Expect(err).NotTo(HaveOccurred())
 
 	return bs
 }, func(bs []byte) {
+	commonTestSetup()
+
 	var sharedSetup sharedSetupData
 	err := json.Unmarshal(bs, &sharedSetup)
 	Expect(err).NotTo(HaveOccurred())
@@ -346,46 +352,44 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	commonTestOrgName = sharedSetup.CommonOrgName
 	defaultAppBitsFile = sharedSetup.DefaultAppBitsFile
 	multiProcessAppBitsFile = sharedSetup.MultiProcessAppBitsFile
+	adminServiceAccount = sharedSetup.AdminServiceAccount
+	adminClient = makeTokenClient(sharedSetup.AdminServiceAccountToken)
 
-	eventuallyTimeoutSeconds := 240
-	customEventuallyTimeoutSeconds := os.Getenv("E2E_EVENTUALLY_TIMEOUT_SECONDS")
-	if customEventuallyTimeoutSeconds != "" {
-		eventuallyTimeoutSeconds, err = strconv.Atoi(customEventuallyTimeoutSeconds)
-		Expect(err).NotTo(HaveOccurred())
-	}
-	SetDefaultEventuallyTimeout(time.Duration(eventuallyTimeoutSeconds) * time.Second)
+	SetDefaultEventuallyTimeout(helpers.EventuallyTimeout())
 	SetDefaultEventuallyPollingInterval(2 * time.Second)
 
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
-
-	commonTestSetup()
 })
 
 var _ = SynchronizedAfterSuite(func() {
 }, func() {
 	os.RemoveAll(assetsTmpDir)
 	deleteOrg(commonTestOrgGUID)
+	serviceAccountFactory.DeleteServiceAccount(adminServiceAccount)
 })
 
 var _ = BeforeEach(func() {
 	correlationId = uuid.NewString()
 })
 
-func makeClient(certEnvVar, tokenEnvVar string) *helpers.CorrelatedRestyClient {
+func makeCertClientForUserName(userName string, validFor time.Duration) *helpers.CorrelatedRestyClient {
 	GinkgoHelper()
 
-	cert := os.Getenv(certEnvVar)
-	if cert != "" {
-		return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthScheme("ClientCert").SetAuthToken(cert).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+	if _, skip := helpers.GetEKSUserPEM(); skip {
+		Skip("EKS does not support cert users: https://github.com/aws/containers-roadmap/issues/1604#issuecomment-1072660824")
 	}
 
-	token := os.Getenv(tokenEnvVar)
-	if token != "" {
-		return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthToken(token).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-	}
+	return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
+		SetAuthScheme("ClientCert").
+		SetAuthToken(base64.StdEncoding.EncodeToString(helpers.CreateTrustedCertificatePEM(userName, validFor))).
+		SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+}
 
-	Fail(fmt.Sprintf("One of %q or %q should have a value, but they are both empty", certEnvVar, tokenEnvVar))
-	return nil
+func makeTokenClient(token string) *helpers.CorrelatedRestyClient {
+	return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
+		SetAuthScheme("Bearer").
+		SetAuthToken(token).
+		SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
 }
 
 func ensureServerIsUp() {
@@ -1025,14 +1029,6 @@ func addDestinationForRoute(appGUID, routeGUID string) []string {
 func commonTestSetup() {
 	apiServerRoot = helpers.GetRequiredEnvVar("API_SERVER_ROOT")
 	rootNamespace = helpers.GetRequiredEnvVar("ROOT_NAMESPACE")
-	serviceAccountName = fmt.Sprintf("system:serviceaccount:%s:%s", rootNamespace, helpers.GetRequiredEnvVar("E2E_SERVICE_ACCOUNT"))
-	serviceAccountToken = helpers.GetRequiredEnvVar("E2E_SERVICE_ACCOUNT_TOKEN")
-
-	longCertUserName = helpers.GetRequiredEnvVar("E2E_LONGCERT_USER_NAME")
-	longCertPEM = os.Getenv("E2E_LONGCERT_USER_PEM")
-
-	certUserName = helpers.GetRequiredEnvVar("E2E_USER_NAME")
-	certPEM = os.Getenv("E2E_USER_PEM")
 
 	appFQDN = helpers.GetRequiredEnvVar("APP_FQDN")
 
@@ -1041,9 +1037,7 @@ func commonTestSetup() {
 
 	ensureServerIsUp()
 
-	adminClient = makeClient("CF_ADMIN_PEM", "CF_ADMIN_TOKEN")
-	privilegedServiceAccountClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthToken(serviceAccountToken).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-	longCertClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthScheme("ClientCert").SetAuthToken(longCertPEM).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+	serviceAccountFactory = helpers.NewServiceAccountFactory(rootNamespace)
 }
 
 func zipAsset(src string) string {

tests/e2e/orgs_test.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/go-resty/resty/v2"
+	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
@@ -119,12 +121,11 @@ var _ = Describe("Orgs", func() {
 		// that gets called by the CLI on each login.
 		When("The client has a certificate with a long expiry date", func() {
 			BeforeEach(func() {
-				if longCertPEM == "" {
-					Skip("No certificate with a long expiry date provided")
-				}
-				restyClient = longCertClient
-				createOrgRole("organization_manager", longCertUserName, org2GUID)
+				userName := uuid.NewString()
+				restyClient = makeCertClientForUserName(userName, 365*24*time.Hour)
+				createOrgRole("organization_manager", userName, org2GUID)
 			})
+
 			It("returns orgs that the client has a role in and sets an HTTP warning header", func() {
 				Expect(resp).To(HaveRestyStatusCode(http.StatusOK))
 				Expect(resp).To(HaveRestyHeaderWithValue("X-Cf-Warnings", HavePrefix("Warning: The client certificate you provided for user authentication expires at")))

tests/e2e/whoami_test.go

@@ -2,11 +2,15 @@ package e2e_test
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
+	"encoding/pem"
 	"net/http"
+	"time"
 
 	"code.cloudfoundry.org/korifi/tests/helpers"
 	"github.com/go-resty/resty/v2"
+	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -19,15 +23,11 @@ type identityResource struct {
 
 var _ = Describe("WhoAmI", func() {
 	var (
-		client   *resty.Client
+		client   *helpers.CorrelatedRestyClient
 		httpResp *resty.Response
 		result   identityResource
 	)
 
-	BeforeEach(func() {
-		client = resty.New().SetBaseURL(apiServerRoot).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-	})
-
 	JustBeforeEach(func() {
 		var err error
 		httpResp, err = client.R().
@@ -37,13 +37,21 @@ var _ = Describe("WhoAmI", func() {
 	})
 
 	When("authenticating with a Bearer token", func() {
+		var svcAcctName string
+
 		BeforeEach(func() {
-			client = client.SetAuthToken(serviceAccountToken)
+			svcAcctName = uuid.NewString()
+			serviceAccountToken := serviceAccountFactory.CreateServiceAccount(svcAcctName)
+			client = makeTokenClient(serviceAccountToken)
+		})
+
+		AfterEach(func() {
+			serviceAccountFactory.DeleteServiceAccount(svcAcctName)
 		})
 
 		It("returns the user identity", func() {
 			Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
-			Expect(result.Name).To(Equal(serviceAccountName))
+			Expect(result.Name).To(Equal(serviceAccountFactory.FullyQualifiedName(svcAcctName)))
 			Expect(result.Kind).To(Equal(rbacv1.ServiceAccountKind))
 		})
 
@@ -59,16 +67,16 @@ var _ = Describe("WhoAmI", func() {
 	})
 
 	When("authenticating with a client certificate", func() {
+		var userName string
+
 		BeforeEach(func() {
-			if certPEM == "" {
-				Skip("No certificate provided.")
-			}
-			client = client.SetAuthScheme("ClientCert").SetAuthToken(certPEM)
+			userName = uuid.NewString()
+			client = makeCertClientForUserName(userName, time.Hour)
 		})
 
 		It("returns the user identity", func() {
 			Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
-			Expect(result.Name).To(Equal(certUserName))
+			Expect(result.Name).To(Equal(userName))
 			Expect(result.Kind).To(Equal(rbacv1.UserKind))
 		})
 
@@ -84,8 +92,7 @@ var _ = Describe("WhoAmI", func() {
 
 		When("the cert is unauthorized", func() {
 			BeforeEach(func() {
-				unauthorisedCertPEM := base64.StdEncoding.EncodeToString(helpers.CreateCertificatePEM())
-				client = client.SetAuthToken(unauthorisedCertPEM)
+				client = client.SetAuthToken(base64.StdEncoding.EncodeToString(helpers.CreateSelfSignedCertificatePEM()))
 			})
 
 			It("returns an unauthorized error", func() {
@@ -94,7 +101,38 @@ var _ = Describe("WhoAmI", func() {
 		})
 	})
 
+	When("authenticating with a provided certificate", func() {
+		var userName string
+
+		BeforeEach(func() {
+			userPem, ok := helpers.GetEKSUserPEM()
+			if !ok {
+				Skip("EKS user PEM not provided")
+			}
+			client = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
+				SetAuthScheme("ClientCert").
+				SetAuthToken(base64.StdEncoding.EncodeToString([]byte(userPem))).
+				SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+
+			certBytes, _ := pem.Decode([]byte(userPem))
+			Expect(certBytes).NotTo(BeNil())
+
+			cert, err := x509.ParseCertificate(certBytes.Bytes)
+			Expect(err).NotTo(HaveOccurred())
+			userName = cert.Subject.CommonName
+		})
+
+		It("returns the user identity", func() {
+			Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
+			Expect(result.Name).To(Equal(userName))
+			Expect(result.Kind).To(Equal(rbacv1.UserKind))
+		})
+	})
+
 	When("no Authorization header is available in the request", func() {
+		BeforeEach(func() {
+			client = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+		})
 		It("returns unauthorized error", func() {
 			Expect(httpResp).To(HaveRestyStatusCode(http.StatusUnauthorized))
 		})