Generate user accounts on the fly in tests

instead of relying them being created in advance

User management is backed up by creating service accounts on the fly,
and use their tokens to create CF API rest clients.

For tests that verify certificate authentication we generate
certificates on the fly as well. However, on EKS signing certificates
with `client auth` usage is not supported, therefore we skip those
tests. As a matter of fact, those tests have already been skipped on
EKS by simply not injecting the certificate related environment
variables.

All of the above will simplify CI setup significantly.

Co-authored-by: Danail Branekov <danailster@gmail.com>
Co-authored-by: Georgi Sabev <georgethebeatle@gmail.com>

api/authorization/user_client_factory_test.go

@@ -237,7 +237,7 @@ var _ = Describe("Unprivileged User Client Factory", func() {
 
 		When("the cert is not valid on this cluster", func() {
 			BeforeEach(func() {
-				authInfo.CertData = helpers.CreateCertificatePEM()
+				authInfo.CertData = helpers.CreateSelfSignedCertificatePEM()
 			})
 
 			It("creates an unusable client", func() {

tests/e2e/authorization_test.go

@@ -1,7 +1,6 @@
 package e2e_test
 
 import (
-	"crypto/tls"
 	"net/http"
 
 	"code.cloudfoundry.org/korifi/tests/helpers"
@@ -13,9 +12,6 @@ import (
 
 var _ = Describe("Authorization", func() {
 	var (
-		serviceaccountFactory *helpers.ServiceAccountFactory
-		svcAcctName           string
-
 		userName   string
 		userClient *helpers.CorrelatedRestyClient
 
@@ -24,18 +20,12 @@ var _ = Describe("Authorization", func() {
 	)
 
 	BeforeEach(func() {
-		serviceaccountFactory = helpers.NewServiceAccountFactory(rootNamespace)
-		svcAcctName = uuid.NewString()
-
-		userName = "system:serviceaccount:cf:" + svcAcctName
-		userToken := serviceaccountFactory.CreateServiceAccount(svcAcctName)
-		userClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
-			SetAuthToken(userToken).
-			SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+		userName = uuid.NewString()
+		userClient = makeTokenClient(serviceAccountFactory.CreateServiceAccount(userName))
 	})
 
 	AfterEach(func() {
-		serviceaccountFactory.DeleteServiceAccount(svcAcctName)
+		serviceAccountFactory.DeleteServiceAccount(userName)
 	})
 
 	Describe("Unauthorized users", func() {
@@ -77,11 +67,11 @@ var _ = Describe("Authorization", func() {
 		BeforeEach(func() {
 			orgName := generateGUID("org")
 			orgGUID = createOrg(orgName)
-			createOrgRole("organization_user", userName, orgGUID)
+			createOrgRole("organization_user", serviceAccountFactory.FullyQualifiedName(userName), orgGUID)
 
 			spaceName := generateGUID("space")
 			spaceGUID = createSpace(spaceName, orgGUID)
-			createSpaceRole("space_developer", userName, spaceGUID)
+			createSpaceRole("space_developer", serviceAccountFactory.FullyQualifiedName(userName), spaceGUID)
 		})
 
 		AfterEach(func() {

tests/e2e/e2e_suite_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"context"
 	"crypto/tls"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -38,21 +39,12 @@ import (
 var (
 	correlationId string
 
-	adminClient                    *helpers.CorrelatedRestyClient
-	privilegedServiceAccountClient *helpers.CorrelatedRestyClient
-	longCertClient                 *helpers.CorrelatedRestyClient
+	serviceAccountFactory *helpers.ServiceAccountFactory
+	adminServiceAccount   string
+	adminClient           *helpers.CorrelatedRestyClient
 
 	apiServerRoot string
 
-	serviceAccountName  string
-	serviceAccountToken string
-
-	certUserName string
-	certPEM      string
-
-	longCertUserName string
-	longCertPEM      string
-
 	rootNamespace           string
 	appFQDN                 string
 	commonTestOrgGUID       string
@@ -307,14 +299,22 @@ func TestE2E(t *testing.T) {
 }
 
 type sharedSetupData struct {
-	CommonOrgName           string `json:"commonOrgName"`
-	CommonOrgGUID           string `json:"commonOrgGuid"`
-	DefaultAppBitsFile      string `json:"defaultAppBitsFile"`
-	MultiProcessAppBitsFile string `json:"multiProcessAppBitsFile"`
+	CommonOrgName            string `json:"commonOrgName"`
+	CommonOrgGUID            string `json:"commonOrgGuid"`
+	DefaultAppBitsFile       string `json:"defaultAppBitsFile"`
+	MultiProcessAppBitsFile  string `json:"multiProcessAppBitsFile"`
+	AdminServiceAccount      string `json:"admin_service_account"`
+	AdminServiceAccountToken string `json:"admin_service_account_token"`
 }
 
 var _ = SynchronizedBeforeSuite(func() []byte {
 	commonTestSetup()
+
+	adminServiceAccount = uuid.NewString()
+	adminServiceAccountToken := serviceAccountFactory.CreateAdminServiceAccount(adminServiceAccount)
+
+	adminClient = makeTokenClient(adminServiceAccountToken)
+
 	commonTestOrgName = generateGUID("common-test-org")
 	commonTestOrgGUID = createOrg(commonTestOrgName)
 
@@ -329,15 +329,19 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		// The DEFAULT_APP_BITS_PATH and DEFAULT_APP_RESPONSE environment variables are a workaround to allow e2e tests to run
 		// with a different app in these environments.
 		// See https://github.com/cloudfoundry/korifi/issues/2355 for refactoring ideas
-		DefaultAppBitsFile:      zipAsset(helpers.GetDefaultedEnvVar("DEFAULT_APP_BITS_PATH", "../assets/dorifi")),
-		MultiProcessAppBitsFile: zipAsset("../assets/multi-process"),
+		DefaultAppBitsFile:       zipAsset(helpers.GetDefaultedEnvVar("DEFAULT_APP_BITS_PATH", "../assets/dorifi")),
+		MultiProcessAppBitsFile:  zipAsset("../assets/multi-process"),
+		AdminServiceAccount:      adminServiceAccount,
+		AdminServiceAccountToken: adminServiceAccountToken,
 	}
 
 	bs, err := json.Marshal(sharedData)
 	Expect(err).NotTo(HaveOccurred())
 
 	return bs
 }, func(bs []byte) {
+	commonTestSetup()
+
 	var sharedSetup sharedSetupData
 	err := json.Unmarshal(bs, &sharedSetup)
 	Expect(err).NotTo(HaveOccurred())
@@ -346,46 +350,44 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	commonTestOrgName = sharedSetup.CommonOrgName
 	defaultAppBitsFile = sharedSetup.DefaultAppBitsFile
 	multiProcessAppBitsFile = sharedSetup.MultiProcessAppBitsFile
+	adminServiceAccount = sharedSetup.AdminServiceAccount
+	adminClient = makeTokenClient(sharedSetup.AdminServiceAccountToken)
 
-	eventuallyTimeoutSeconds := 240
-	customEventuallyTimeoutSeconds := os.Getenv("E2E_EVENTUALLY_TIMEOUT_SECONDS")
-	if customEventuallyTimeoutSeconds != "" {
-		eventuallyTimeoutSeconds, err = strconv.Atoi(customEventuallyTimeoutSeconds)
-		Expect(err).NotTo(HaveOccurred())
-	}
-	SetDefaultEventuallyTimeout(time.Duration(eventuallyTimeoutSeconds) * time.Second)
+	SetDefaultEventuallyTimeout(helpers.EventuallyTimeout())
 	SetDefaultEventuallyPollingInterval(2 * time.Second)
 
 	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
-
-	commonTestSetup()
 })
 
 var _ = SynchronizedAfterSuite(func() {
 }, func() {
 	os.RemoveAll(assetsTmpDir)
 	deleteOrg(commonTestOrgGUID)
+	serviceAccountFactory.DeleteServiceAccount(adminServiceAccount)
 })
 
 var _ = BeforeEach(func() {
 	correlationId = uuid.NewString()
 })
 
-func makeClient(certEnvVar, tokenEnvVar string) *helpers.CorrelatedRestyClient {
+func makeCertClientForUserName(userName string, validFor time.Duration) *helpers.CorrelatedRestyClient {
 	GinkgoHelper()
 
-	cert := os.Getenv(certEnvVar)
-	if cert != "" {
-		return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthScheme("ClientCert").SetAuthToken(cert).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+	if os.Getenv("CLUSTER_TYPE") == "EKS" {
+		Skip("EKS does not support cert users: https://github.com/aws/containers-roadmap/issues/1604#issuecomment-1072660824")
 	}
 
-	token := os.Getenv(tokenEnvVar)
-	if token != "" {
-		return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthToken(token).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-	}
+	return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
+		SetAuthScheme("ClientCert").
+		SetAuthToken(base64.StdEncoding.EncodeToString(helpers.CreateTrustedCertificatePEM(userName, validFor))).
+		SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+}
 
-	Fail(fmt.Sprintf("One of %q or %q should have a value, but they are both empty", certEnvVar, tokenEnvVar))
-	return nil
+func makeTokenClient(token string) *helpers.CorrelatedRestyClient {
+	return helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).
+		SetAuthScheme("Bearer").
+		SetAuthToken(token).
+		SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
 }
 
 func ensureServerIsUp() {
@@ -1025,14 +1027,6 @@ func addDestinationForRoute(appGUID, routeGUID string) []string {
 func commonTestSetup() {
 	apiServerRoot = helpers.GetRequiredEnvVar("API_SERVER_ROOT")
 	rootNamespace = helpers.GetRequiredEnvVar("ROOT_NAMESPACE")
-	serviceAccountName = fmt.Sprintf("system:serviceaccount:%s:%s", rootNamespace, helpers.GetRequiredEnvVar("E2E_SERVICE_ACCOUNT"))
-	serviceAccountToken = helpers.GetRequiredEnvVar("E2E_SERVICE_ACCOUNT_TOKEN")
-
-	longCertUserName = helpers.GetRequiredEnvVar("E2E_LONGCERT_USER_NAME")
-	longCertPEM = os.Getenv("E2E_LONGCERT_USER_PEM")
-
-	certUserName = helpers.GetRequiredEnvVar("E2E_USER_NAME")
-	certPEM = os.Getenv("E2E_USER_PEM")
 
 	appFQDN = helpers.GetRequiredEnvVar("APP_FQDN")
 
@@ -1041,9 +1035,7 @@ func commonTestSetup() {
 
 	ensureServerIsUp()
 
-	adminClient = makeClient("CF_ADMIN_PEM", "CF_ADMIN_TOKEN")
-	privilegedServiceAccountClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthToken(serviceAccountToken).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-	longCertClient = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetAuthScheme("ClientCert").SetAuthToken(longCertPEM).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+	serviceAccountFactory = helpers.NewServiceAccountFactory(rootNamespace)
 }
 
 func zipAsset(src string) string {

tests/e2e/orgs_test.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 	"net/http"
 	"sync"
+	"time"
 
 	"github.com/go-resty/resty/v2"
+	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
@@ -119,12 +121,11 @@ var _ = Describe("Orgs", func() {
 		// that gets called by the CLI on each login.
 		When("The client has a certificate with a long expiry date", func() {
 			BeforeEach(func() {
-				if longCertPEM == "" {
-					Skip("No certificate with a long expiry date provided")
-				}
-				restyClient = longCertClient
-				createOrgRole("organization_manager", longCertUserName, org2GUID)
+				userName := uuid.NewString()
+				restyClient = makeCertClientForUserName(userName, 365*24*time.Hour)
+				createOrgRole("organization_manager", userName, org2GUID)
 			})
+
 			It("returns orgs that the client has a role in and sets an HTTP warning header", func() {
 				Expect(resp).To(HaveRestyStatusCode(http.StatusOK))
 				Expect(resp).To(HaveRestyHeaderWithValue("X-Cf-Warnings", HavePrefix("Warning: The client certificate you provided for user authentication expires at")))

tests/e2e/whoami_test.go

@@ -4,9 +4,11 @@ import (
 	"crypto/tls"
 	"encoding/base64"
 	"net/http"
+	"time"
 
 	"code.cloudfoundry.org/korifi/tests/helpers"
 	"github.com/go-resty/resty/v2"
+	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -19,15 +21,11 @@ type identityResource struct {
 
 var _ = Describe("WhoAmI", func() {
 	var (
-		client   *resty.Client
+		client   *helpers.CorrelatedRestyClient
 		httpResp *resty.Response
 		result   identityResource
 	)
 
-	BeforeEach(func() {
-		client = resty.New().SetBaseURL(apiServerRoot).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-	})
-
 	JustBeforeEach(func() {
 		var err error
 		httpResp, err = client.R().
@@ -37,13 +35,21 @@ var _ = Describe("WhoAmI", func() {
 	})
 
 	When("authenticating with a Bearer token", func() {
+		var svcAcctName string
+
 		BeforeEach(func() {
-			client = client.SetAuthToken(serviceAccountToken)
+			svcAcctName = uuid.NewString()
+			serviceAccountToken := serviceAccountFactory.CreateServiceAccount(svcAcctName)
+			client = makeTokenClient(serviceAccountToken)
+		})
+
+		AfterEach(func() {
+			serviceAccountFactory.DeleteServiceAccount(svcAcctName)
 		})
 
 		It("returns the user identity", func() {
 			Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
-			Expect(result.Name).To(Equal(serviceAccountName))
+			Expect(result.Name).To(Equal(serviceAccountFactory.FullyQualifiedName(svcAcctName)))
 			Expect(result.Kind).To(Equal(rbacv1.ServiceAccountKind))
 		})
 
@@ -59,16 +65,16 @@ var _ = Describe("WhoAmI", func() {
 	})
 
 	When("authenticating with a client certificate", func() {
+		var userName string
+
 		BeforeEach(func() {
-			if certPEM == "" {
-				Skip("No certificate provided.")
-			}
-			client = client.SetAuthScheme("ClientCert").SetAuthToken(certPEM)
+			userName = uuid.NewString()
+			client = makeCertClientForUserName(userName, time.Hour)
 		})
 
 		It("returns the user identity", func() {
 			Expect(httpResp).To(HaveRestyStatusCode(http.StatusOK))
-			Expect(result.Name).To(Equal(certUserName))
+			Expect(result.Name).To(Equal(userName))
 			Expect(result.Kind).To(Equal(rbacv1.UserKind))
 		})
 
@@ -84,8 +90,7 @@ var _ = Describe("WhoAmI", func() {
 
 		When("the cert is unauthorized", func() {
 			BeforeEach(func() {
-				unauthorisedCertPEM := base64.StdEncoding.EncodeToString(helpers.CreateCertificatePEM())
-				client = client.SetAuthToken(unauthorisedCertPEM)
+				client = client.SetAuthToken(base64.StdEncoding.EncodeToString(helpers.CreateSelfSignedCertificatePEM()))
 			})
 
 			It("returns an unauthorized error", func() {
@@ -95,6 +100,9 @@ var _ = Describe("WhoAmI", func() {
 	})
 
 	When("no Authorization header is available in the request", func() {
+		BeforeEach(func() {
+			client = helpers.NewCorrelatedRestyClient(apiServerRoot, getCorrelationId).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+		})
 		It("returns unauthorized error", func() {
 			Expect(httpResp).To(HaveRestyStatusCode(http.StatusUnauthorized))
 		})