From a85d91004205ea3549f96f266c9f4a8b81d98578 Mon Sep 17 00:00:00 2001 From: KBOCPDocs Date: Tue, 16 Apr 2024 12:09:43 -0400 Subject: [PATCH] PostgreSQL integration --- Makefile | 3 +- README.md | 38 +- api/v1alpha1/olsconfig_types.go | 41 +- api/v1alpha1/zz_generated.deepcopy.go | 40 +- ...tspeed-operator.clusterserviceversion.yaml | 32 +- .../ols.openshift.io_olsconfigs.yaml | 38 +- cmd/main.go | 38 +- .../bases/ols.openshift.io_olsconfigs.yaml | 38 +- config/default/kustomization.yaml | 3 + ...tspeed-operator.clusterserviceversion.yaml | 31 +- config/rbac/role.yaml | 1 + internal/controller/constants.go | 97 +++- internal/controller/errors.go | 24 + .../controller/ols_app_postgres_assets.go | 342 ++++++++++++++ .../ols_app_postgres_assets_test.go | 431 ++++++++++++++++++ .../ols_app_postgres_reconciliator.go | 213 +++++++++ .../ols_app_postgres_reconciliator_test.go | 169 +++++++ internal/controller/ols_app_server_assets.go | 69 ++- .../controller/ols_app_server_assets_test.go | 225 +++++++-- .../controller/ols_app_server_deployment.go | 52 ++- .../ols_app_server_reconciliator.go | 30 +- internal/controller/olsconfig_controller.go | 28 +- internal/controller/suite_test.go | 8 +- internal/controller/types.go | 37 +- internal/controller/utils.go | 26 +- test/e2e/assets.go | 10 +- test/e2e/client.go | 2 +- 27 files changed, 1761 insertions(+), 305 deletions(-) create mode 100644 internal/controller/ols_app_postgres_assets.go create mode 100644 internal/controller/ols_app_postgres_assets_test.go create mode 100644 internal/controller/ols_app_postgres_reconciliator.go create mode 100644 internal/controller/ols_app_postgres_reconciliator_test.go diff --git a/Makefile b/Makefile index 2d0ab74a..e59fee3f 100644 --- a/Makefile +++ b/Makefile @@ -184,11 +184,12 @@ build: manifests generate fmt vet ## Build manager binary. go build -o bin/manager cmd/main.go LIGHTSPEED_SERVICE_IMG ?= quay.io/openshift-lightspeed/lightspeed-service-api:latest +LIGHTSPEED_SERVICE_POSTGRES_IMG ?= registry.redhat.io/rhel9/postgresql-16@sha256:0d52a138698bf42dbbf5b9e4f35f4925b6155eef9f447e436c6db1a7ae304239 CONSOLE_PLUGIN_IMG ?= quay.io/openshift-lightspeed/lightspeed-console-plugin:latest .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. #TODO: Update DB - go run ./cmd/main.go --service-image="$(LIGHTSPEED_SERVICE_IMG)" --console-image="$(CONSOLE_PLUGIN_IMG)" + go run ./cmd/main.go --service-image="$(LIGHTSPEED_SERVICE_IMG)" --postgres-image="$(LIGHTSPEED_SERVICE_POSTGRES_IMG)" --console-image="$(CONSOLE_PLUGIN_IMG)" # If you wish built the manager image targeting other platforms you can use the --platform flag. # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. diff --git a/README.md b/README.md index d7f16aec..caa411d7 100644 --- a/README.md +++ b/README.md @@ -122,10 +122,10 @@ spec: url: "https://myendpoint.openai.azure.com/" ols: conversationCache: - redis: - maxMemory: 2000mb - maxMemoryPolicy: allkeys-lru - type: redis + postgres: + sharedBuffers: 256MB + maxConnections: 2000 + type: postgres defaultModel: gpt-3.5-turbo defaultProvider: openai logLevel: INFO @@ -187,22 +187,25 @@ openshift-service-ca.crt 1 33m ➜ oc get services -n openshift-lightspeed NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -lightspeed-app-server ClusterIP 172.31.165.151 8443/TCP 22m -lightspeed-console-plugin ClusterIP 172.31.158.29 9443/TCP 29m -lightspeed-operator-controller-manager-service ClusterIP 172.31.63.140 8443/TCP 24m +lightspeed-app-server ClusterIP 172.30.176.179 8080/TCP 4m47s +lightspeed-postgres-server ClusterIP 172.30.85.42 6379/TCP 4m47s +lightspeed-operator-controller-manager-metrics-service ClusterIP 172.30.35.253 8443/TCP 4m47s +lightspeed-console-plugin ClusterIP 172.31.158.29 9443/TCP 29m +lightspeed-operator-controller-manager-service ClusterIP 172.31.63.140 8443/TCP 24m ➜ oc get deployments -n openshift-lightspeed NAME READY UP-TO-DATE AVAILABLE AGE -lightspeed-app-server 1/1 1 1 23m +lightspeed-app-server 1/1 1 1 7m5s +lightspeed-postgres-server 1/1 1 1 7m5s +lightspeed-operator-controller-manager 1/1 1 1 2d15h lightspeed-console-plugin 2/2 2 2 30m -lightspeed-operator-controller-manager 1/1 1 1 25m ➜ oc get pods -n openshift-lightspeed NAME READY STATUS RESTARTS AGE +lightspeed-operator-controller-manager-7c849865ff-9vwj9 2/2 Running 0 7m19s +lightspeed-postgres-server-7b75497676-np7zk 1/1 Running 0 6m47s lightspeed-app-server-97c9c6d96-6tv6j 2/2 Running 0 23m -lilightspeed-console-plugin-7f6cd7c9fd-6lp7x 1/1 Running 0 30m lightspeed-console-plugin-7f6cd7c9fd-wctj8 1/1 Running 0 30m -lightspeed-operator-controller-manager-69585cc7fc-xltpc 1/1 Running 0 26m ➜ oc logs lightspeed-app-server-f7fd6cf6-k7s7p -n openshift-lightspeed 2024-02-02 12:00:06,982 [ols.app.main:main.py:29] INFO: Embedded Gradio UI is disabled. To enable set enable_dev_ui: true in the dev section of the configuration file @@ -212,6 +215,17 @@ INFO: Application startup complete. INFO: Uvicorn running on http://0.0.0.0:8080 (Press CTRL+C to quit) ``` +#### Postgres Secret Management +By default postgres server spins up with a randomly generated password located in the secret `lightspeed-postgres-secret`. One can go edit password their password to a desired value to get it reflected across the system. In addition to that postgres secret name can also be explicitly specified in cluster CR as shown in the below example. +``` +conversationCache: + postgres: + sharedBuffers: "256MB" + maxConnections: 2000 + credentialsSecret: xyz + type: postgres +``` + ### Modifying the API definitions If you have updated the API definitions, you must update the CRD manifests with the following command @@ -255,7 +269,7 @@ When using Visual Studio Code, we can use the debugger settings below to execute ### End to End tests -To run the end to end tests with a Openshift cluster, we need to have a running operator in the namespace `openshift-lightspeed`. +To run the end to end tests with a OpenShift cluster, we need to have a running operator in the namespace `openshift-lightspeed`. Please refer to the section [Running on the cluster](#running-on-the-cluster). Then we should set 2 environment variables: diff --git a/api/v1alpha1/olsconfig_types.go b/api/v1alpha1/olsconfig_types.go index 0f0ec615..4c292b90 100644 --- a/api/v1alpha1/olsconfig_types.go +++ b/api/v1alpha1/olsconfig_types.go @@ -20,7 +20,6 @@ import ( configv1 "github.com/openshift/api/config/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -145,39 +144,41 @@ type ConsoleContainerConfig struct { CAcertificate string `json:"caCertificate,omitempty"` } -// +kubebuilder:validation:Enum=redis +// +kubebuilder:validation:Enum=postgres type CacheType string const ( - Redis CacheType = "redis" + Postgres CacheType = "postgres" ) // ConversationCacheSpec defines the desired state of OLS conversation cache. type ConversationCacheSpec struct { - // Conversation cache type. Default: "redis" - // +kubebuilder:default=redis + // Conversation cache type. Default: "postgres" + // +kubebuilder:default=postgres // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Cache Type" Type CacheType `json:"type,omitempty"` // +optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Redis" - Redis RedisSpec `json:"redis,omitempty"` + Postgres PostgresSpec `json:"postgres,omitempty"` } -// RedisSpec defines the desired state of Redis. -type RedisSpec struct { - // Secret that holds redis credentials - // +kubebuilder:default="lightspeed-redis-secret" - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Credentials secret" +// PostgresSpec defines the desired state of Postgres. +type PostgresSpec struct { + // Postgres user name + // +kubebuilder:default="postgres" + User string `json:"user,omitempty"` + // Postgres database name + // +kubebuilder:default="postgres" + DbName string `json:"dbName,omitempty"` + // Secret that holds postgres credentials + // +kubebuilder:default="lightspeed-postgres-secret" CredentialsSecret string `json:"credentialsSecret,omitempty"` - // Redis maxmemory + // Postgres sharedbuffers // +kubebuilder:validation:XIntOrString - // +kubebuilder:default="1024mb" - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Max Memory" - MaxMemory *intstr.IntOrString `json:"maxMemory,omitempty"` - // Redis maxmemory policy. Default: "allkeys-lru" - // +kubebuilder:default=allkeys-lru - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Max Memory Policy" - MaxMemoryPolicy string `json:"maxMemoryPolicy,omitempty"` + // +kubebuilder:default="256MB" + SharedBuffers string `json:"sharedBuffers,omitempty"` + // Postgres maxconnections. Default: "2000" + // +kubebuilder:default=2000 + MaxConnections int `json:"maxConnections,omitempty"` } // QueryFiltersSpec defines filters to manipulate questions/queries. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index aded9524..0918000a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -104,7 +103,7 @@ func (in *ConsoleContainerConfig) DeepCopy() *ConsoleContainerConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConversationCacheSpec) DeepCopyInto(out *ConversationCacheSpec) { *out = *in - in.Redis.DeepCopyInto(&out.Redis) + out.Postgres = in.Postgres } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConversationCacheSpec. @@ -330,7 +329,7 @@ func (in *OLSDataCollectorSpec) DeepCopy() *OLSDataCollectorSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OLSSpec) DeepCopyInto(out *OLSSpec) { *out = *in - in.ConversationCache.DeepCopyInto(&out.ConversationCache) + out.ConversationCache = in.ConversationCache in.DeploymentConfig.DeepCopyInto(&out.DeploymentConfig) if in.QueryFilters != nil { in, out := &in.QueryFilters, &out.QueryFilters @@ -365,6 +364,21 @@ func (in *OLSSpec) DeepCopy() *OLSSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostgresSpec. +func (in *PostgresSpec) DeepCopy() *PostgresSpec { + if in == nil { + return nil + } + out := new(PostgresSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = *in @@ -406,26 +420,6 @@ func (in *QueryFiltersSpec) DeepCopy() *QueryFiltersSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RedisSpec) DeepCopyInto(out *RedisSpec) { - *out = *in - if in.MaxMemory != nil { - in, out := &in.MaxMemory, &out.MaxMemory - *out = new(intstr.IntOrString) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisSpec. -func (in *RedisSpec) DeepCopy() *RedisSpec { - if in == nil { - return nil - } - out := new(RedisSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { *out = *in diff --git a/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml b/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml index b799d683..c4a20cc4 100644 --- a/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml +++ b/bundle/manifests/lightspeed-operator.clusterserviceversion.yaml @@ -127,18 +127,27 @@ spec: path: ols.additionalCAConfigMapRef x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - - displayName: Redis - path: ols.conversationCache.redis - - description: Secret that holds redis credentials + - displayName: Postgres + path: ols.conversationCache.postgres + - description: Secret that holds Postgres credentials displayName: Credentials secret - path: ols.conversationCache.redis.credentialsSecret - - description: Redis maxmemory - displayName: Max Memory - path: ols.conversationCache.redis.maxMemory - - description: 'Redis maxmemory policy. Default: "allkeys-lru"' - displayName: Max Memory Policy - path: ols.conversationCache.redis.maxMemoryPolicy - - description: 'Conversation cache type. Default: "redis"' + path: ols.conversationCache.postgres.credentialsSecret + - description: Postgres database name + displayName: Database name + path: ols.conversationCache.postgres.dbName + - description: Postgres max connections + displayName: Max connections + path: ols.conversationCache.postgres.maxConnections + - description: Postgres max connections + displayName: Max connections + path: ols.conversationCache.postgres.maxConnections + - description: Postgres shared buffers + displayName: Shared buffers + path: ols.conversationCache.postgres.sharedBuffers + - description: Postgres user name + displayName: User name + path: ols.conversationCache.postgres.user + - description: 'Conversation cache type. Default: "postgres"' displayName: Cache Type path: ols.conversationCache.type - description: Default model for usage @@ -503,6 +512,7 @@ spec: verbs: - create - delete + - deletecollection - get - list - patch diff --git a/bundle/manifests/ols.openshift.io_olsconfigs.yaml b/bundle/manifests/ols.openshift.io_olsconfigs.yaml index edd0176b..8b33c222 100644 --- a/bundle/manifests/ols.openshift.io_olsconfigs.yaml +++ b/bundle/manifests/ols.openshift.io_olsconfigs.yaml @@ -412,30 +412,36 @@ spec: conversationCache: description: Conversation cache settings properties: - redis: - description: RedisSpec defines the desired state of Redis. + postgres: + description: PostgresSpec defines the desired state of Postgres. properties: credentialsSecret: - default: lightspeed-redis-secret - description: Secret that holds redis credentials + default: lightspeed-postgres-secret + description: Secret that holds postgres credentials + type: string + dbName: + default: postgres + description: Postgres database name + type: string + maxConnections: + default: 2000 + description: 'Postgres maxconnections. Default: "2000"' + type: integer + sharedBuffers: + default: 256MB + description: Postgres sharedbuffers type: string - maxMemory: - anyOf: - - type: integer - - type: string - default: 1024mb - description: Redis maxmemory x-kubernetes-int-or-string: true - maxMemoryPolicy: - default: allkeys-lru - description: 'Redis maxmemory policy. Default: "allkeys-lru"' + user: + default: postgres + description: Postgres user name type: string type: object type: - default: redis - description: 'Conversation cache type. Default: "redis"' + default: postgres + description: 'Conversation cache type. Default: "postgres"' enum: - - redis + - postgres type: string type: object defaultModel: diff --git a/cmd/main.go b/cmd/main.go index 15c12cbe..2f269142 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -62,9 +62,8 @@ var ( // The default images of operands defaultImages = map[string]string{ "lightspeed-service": controller.OLSAppServerImageDefault, - // TODO: Update DB - //"lightspeed-service-redis": controller.RedisServerImageDefault, - "console-plugin": controller.ConsoleUIImageDefault, + "postgres-image": controller.PostgresServerImageDefault, + "console-plugin": controller.ConsoleUIImageDefault, } ) @@ -79,20 +78,20 @@ func init() { //+kubebuilder:scaffold:scheme } -// validateImages overides the default images with the serviceImage and consoleImage provided by the user -// if the images are not provided, the default images are used. -func validateImages(serviceImage string, consoleImage string) (map[string]string, error) { +// overrideImages overides the default images with the images provided by the user +// if an images is not provided, the default is used. +func overrideImages(serviceImage string, consoleImage string, postgresImage string) map[string]string { res := defaultImages - if serviceImage == "" && consoleImage == "" { - return res, nil - } if serviceImage != "" { res["lightspeed-service"] = serviceImage } if consoleImage != "" { res["console-plugin"] = consoleImage } - return res, nil + if postgresImage != "" { + res["postgres-image"] = postgresImage + } + return res } func listImages() []string { @@ -119,6 +118,7 @@ func main() { var metricsClientCA string var serviceImage string var consoleImage string + var postgresImage string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, @@ -132,6 +132,7 @@ func main() { flag.StringVar(&caCertPath, "ca-cert", controller.OperatorCACertPathDefault, "The path to the CA certificate file.") flag.StringVar(&serviceImage, "service-image", controller.OLSAppServerImageDefault, "The image of the lightspeed-service container.") flag.StringVar(&consoleImage, "console-image", controller.ConsoleUIImageDefault, "The image of the console-plugin container.") + flag.StringVar(&postgresImage, "postgres-image", controller.PostgresServerImageDefault, "The image of the PostgreSQL server.") opts := zap.Options{ Development: true, } @@ -140,11 +141,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - imagesMap, err := validateImages(serviceImage, consoleImage) - if err != nil { - setupLog.Error(err, "invalid images") - os.Exit(1) - } + imagesMap := overrideImages(serviceImage, consoleImage, postgresImage) setupLog.Info("Images setting loaded", "images", listImages()) setupLog.Info("Starting the operator", "metricsAddr", metricsAddr, "probeAddr", probeAddr, "reconcilerIntervalMinutes", reconcilerIntervalMinutes, "certDir", certDir, "certName", certName, "keyName", keyName) @@ -247,12 +244,11 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Options: controller.OLSConfigReconcilerOptions{ - LightspeedServiceImage: imagesMap["lightspeed-service"], - ConsoleUIImage: imagesMap["console-plugin"], - // TODO: Update DB - //LightspeedServiceRedisImage: imagesMap["lightspeed-service-redis"], - Namespace: controller.OLSNamespaceDefault, - ReconcileInterval: time.Duration(reconcilerIntervalMinutes) * time.Minute, + LightspeedServiceImage: imagesMap["lightspeed-service"], + ConsoleUIImage: imagesMap["console-plugin"], + LightspeedServicePostgresImage: imagesMap["postgres-image"], + Namespace: controller.OLSNamespaceDefault, + ReconcileInterval: time.Duration(reconcilerIntervalMinutes) * time.Minute, }, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "OLSConfig") diff --git a/config/crd/bases/ols.openshift.io_olsconfigs.yaml b/config/crd/bases/ols.openshift.io_olsconfigs.yaml index ac23f48c..5315eb0f 100644 --- a/config/crd/bases/ols.openshift.io_olsconfigs.yaml +++ b/config/crd/bases/ols.openshift.io_olsconfigs.yaml @@ -412,30 +412,36 @@ spec: conversationCache: description: Conversation cache settings properties: - redis: - description: RedisSpec defines the desired state of Redis. + postgres: + description: PostgresSpec defines the desired state of Postgres. properties: credentialsSecret: - default: lightspeed-redis-secret - description: Secret that holds redis credentials + default: lightspeed-postgres-secret + description: Secret that holds postgres credentials + type: string + dbName: + default: postgres + description: Postgres database name + type: string + maxConnections: + default: 2000 + description: 'Postgres maxconnections. Default: "2000"' + type: integer + sharedBuffers: + default: 256MB + description: Postgres sharedbuffers type: string - maxMemory: - anyOf: - - type: integer - - type: string - default: 1024mb - description: Redis maxmemory x-kubernetes-int-or-string: true - maxMemoryPolicy: - default: allkeys-lru - description: 'Redis maxmemory policy. Default: "allkeys-lru"' + user: + default: postgres + description: Postgres user name type: string type: object type: - default: redis - description: 'Conversation cache type. Default: "redis"' + default: postgres + description: 'Conversation cache type. Default: "postgres"' enum: - - redis + - postgres type: string type: object defaultModel: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 3ede8014..016f5ff2 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -37,6 +37,9 @@ patches: - op: add path: /spec/template/spec/containers/0/args/- value: --console-image=registry.redhat.io/openshift-lightspeed-tech-preview/lightspeed-console-plugin-rhel9@sha256:0feb8141ef4a029e406b1aa2ac5f946bef1d3b4187d1ec3207d69416e5f84180 + - op: add + path: /spec/template/spec/containers/0/args/- + value: --postgres-image=registry.redhat.io/rhel9/postgresql-16@sha256:0d52a138698bf42dbbf5b9e4f35f4925b6155eef9f447e436c6db1a7ae304239 target: group: apps version: v1 diff --git a/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml b/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml index 43a79dd8..28f21654 100644 --- a/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/lightspeed-operator.clusterserviceversion.yaml @@ -95,18 +95,27 @@ spec: path: ols.additionalCAConfigMapRef x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - - displayName: Redis - path: ols.conversationCache.redis - - description: Secret that holds redis credentials + - displayName: Postgres + path: ols.conversationCache.postgres + - description: Secret that holds Postgres credentials displayName: Credentials secret - path: ols.conversationCache.redis.credentialsSecret - - description: Redis maxmemory - displayName: Max Memory - path: ols.conversationCache.redis.maxMemory - - description: 'Redis maxmemory policy. Default: "allkeys-lru"' - displayName: Max Memory Policy - path: ols.conversationCache.redis.maxMemoryPolicy - - description: 'Conversation cache type. Default: "redis"' + path: ols.conversationCache.postgres.credentialsSecret + - description: Postgres database name + displayName: Database name + path: ols.conversationCache.postgres.dbName + - description: Postgres max connections + displayName: Max connections + path: ols.conversationCache.postgres.maxConnections + - description: Postgres max connections + displayName: Max connections + path: ols.conversationCache.postgres.maxConnections + - description: Postgres shared buffers + displayName: Shared buffers + path: ols.conversationCache.postgres.sharedBuffers + - description: Postgres user name + displayName: User name + path: ols.conversationCache.postgres.user + - description: 'Conversation cache type. Default: "postgres"' displayName: Cache Type path: ols.conversationCache.type - description: Default model for usage diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index aedcbd06..e973a2b2 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -133,6 +133,7 @@ rules: verbs: - create - delete + - deletecollection - get - list - patch diff --git a/internal/controller/constants.go b/internal/controller/constants.go index fa725c23..2ee366d4 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -28,6 +28,8 @@ const ( /*** application server configuration file ***/ // OLSConfigName is the name of the OLSConfig configmap OLSConfigCmName = "olsconfig" + // OLSCAConfigMap is the name of the OLS TLS ca certificate configmap + OLSCAConfigMap = "openshift-service-ca.crt" // OLSNamespaceDefault is the default namespace for OLS OLSNamespaceDefault = "openshift-lightspeed" // OLSAppServerServiceAccountName is the name of service account running the application server @@ -66,7 +68,6 @@ const ( // CertBundleDir is the path of the volume for the certificate bundle CertBundleDir = "cert-bundle" - // Image of the OLS application redis server // OLSConfigHashKey is the key of the hash value of the OLSConfig configmap OLSConfigHashKey = "hash/olsconfig" // LLMProviderHashKey is the key of the hash value of OLS LLM provider credentials consolidated @@ -90,18 +91,14 @@ const ( // #nosec G101 ServingCertSecretAnnotationKey = "service.beta.openshift.io/serving-cert-secret-name" /*** state cache keys ***/ - // OLSAppTLSHashStateCacheKey is the key of the hash value of the OLS App TLS certificates - OLSAppTLSHashStateCacheKey = "olsapptls-hash" - // OLSConfigHashStateCacheKey is the key of the hash value of the OLSConfig configmap - OLSConfigHashStateCacheKey = "olsconfigmap-hash" - // OLSConsoleTLSHashStateCacheKey is the key of the hash value of the OLS Console TLS certificates - OLSConsoleTLSHashStateCacheKey = "olsconsoletls-hash" - // LLMProviderHashStateCacheKey is the key of the hash value of OLS LLM provider credentials consolidated + AzureOpenAIType = "azure_openai" + OLSConfigHashStateCacheKey = "olsconfigmap-hash" LLMProviderHashStateCacheKey = "llmprovider-hash" // AzureOpenAIType is the name of the Azure OpenAI provider type AzureOpenAIType = "azure_openai" // AdditionalCAHashStateCacheKey is the key of the hash value of the additional CA certificates in the state cache AdditionalCAHashStateCacheKey = "additionalca-hash" + /*** console UI plugin ***/ // ConsoleUIConfigMapName is the name of the console UI nginx configmap ConsoleUIConfigMapName = "lightspeed-console-plugin" @@ -112,7 +109,7 @@ const ( // ConsoleUIDeploymentName is the name of the console UI deployment ConsoleUIDeploymentName = "lightspeed-console-plugin" // ConsoleUIImage is the image of the console UI plugin - ConsoleUIImageDefault = "quay.io/openshift-lightspeed/lightspeed-console-plugin:latest" + ConsoleUIImageDefault = "quay.io/openshift/lightspeed-console-plugin:latest" // ConsoleUIHTTPSPort is the port number of the console UI service ConsoleUIHTTPSPort = 9443 // ConsoleUIPluginName is the name of the console UI plugin @@ -124,12 +121,92 @@ const ( // ConsoleProxyAlias is the alias of the console proxy // The console backend exposes following proxy endpoint: /api/proxy/plugin///? ConsoleProxyAlias = "ols" + // OLSConsoleTLSHashStateCacheKey is the key of the hash value of the OLS Console TLS certificates + OLSConsoleTLSHashStateCacheKey = "olsconsoletls-hash" /*** watchers ***/ WatcherAnnotationKey = "ols.openshift.io/watcher" + + /*** Postgres Constants ***/ + // PostgresCAVolume is the name of the OLS postgres TLS ca certificate volume name + PostgresCAVolume = "cm-olspostgresca" + // PostgresDeploymentName is the name of OLS application postgres deployment + PostgresDeploymentName = "lightspeed-postgres-server" + // PostgresSecretKeyName is the name of the key holding postgres server secret + PostgresSecretKeyName = "password" + // Image of the OLS application postgres server + PostgresServerImageDefault = "registry.redhat.io/rhel9/postgresql-16@sha256:0d52a138698bf42dbbf5b9e4f35f4925b6155eef9f447e436c6db1a7ae304239" + // PostgresDefaultUser is the default user name for postgres + PostgresDefaultUser = "postgres" + // PostgresDefaultDbName is the default db name for postgres + PostgresDefaultDbName = "postgres" + // PostgresConfigHashKey is the key of the hash value of the OLS's postgres config + PostgresConfigHashKey = "hash/olspostgresconfig" + // PostgresSecretHashKey is the key of the hash value of OLS Postgres secret + // #nosec G101 + PostgresSecretHashKey = "hash/postgres-secret" + // PostgresServiceName is the name of OLS application postgres server service + PostgresServiceName = "lightspeed-postgres-server" + // PostgresSecretName is the name of OLS application postgres secret + PostgresSecretName = "lightspeed-postgres-secret" + // PostgresCertsSecretName is the name of the postgres certs secret + PostgresCertsSecretName = "lightspeed-postgres-certs" + // PostgresBootstrapSecretName is the name of the postgres bootstrap secret + // #nosec G101 + PostgresBootstrapSecretName = "lightspeed-postgres-bootstrap" + // PostgresBootstrapVolumeMountPath is the path of bootstrap volume mount + PostgresBootstrapVolumeMountPath = "/usr/share/container-scripts/postgresql/start/create-extensions.sh" + // PostgresExtensionScript is the name of the postgres extensions script + PostgresExtensionScript = "create-extensions.sh" + // PostgresConfigMap is the name of the postgres config map + PostgresConfigMap = "lightspeed-postgres-conf" + // PostgresConfigVolumeMountPath is the path of postgres configuration volume mount + PostgresConfigVolumeMountPath = "/usr/share/pgsql/postgresql.conf.sample" + // PostgresConfig is the name of postgres configuration used to start the server + PostgresConfig = "postgresql.conf.sample" + // PostgresDataVolume is the name of postgres data volume + PostgresDataVolume = "postgres-data" + // PostgresDataVolumeMountPath is the path of postgres data volume mount + PostgresDataVolumeMountPath = "/var/lib/pgsql/data" + // PostgresServicePort is the port number of the OLS postgres server service + PostgresServicePort = 5432 + // PostgresSharedBuffers is the share buffers value for postgres cache + PostgresSharedBuffers = "256MB" + // PostgresMaxConnections is the max connections values for postgres cache + PostgresMaxConnections = 2000 + // PostgresDefaultSSLMode is the default ssl mode for postgres + PostgresDefaultSSLMode = "require" + // PostgresBootStrapScriptContent is the postgres's bootstrap script content + PostgresBootStrapScriptContent = ` +#!/bin/bash + +cat /var/lib/pgsql/data/userdata/postgresql.conf + +echo "attempting to create pg_trgm extension if it does not exist" + +_psql () { psql --set ON_ERROR_STOP=1 "$@" ; } + +echo "CREATE EXTENSION IF NOT EXISTS pg_trgm;" | _psql -d $POSTGRESQL_DATABASE +` + // PostgresConfigMapContent is the postgres's config content + PostgresConfigMapContent = ` +huge_pages = off +ssl = on +ssl_cert_file = '/etc/certs/tls.crt' +ssl_key_file = '/etc/certs/tls.key' +ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt' +` + /*** state cache keys ***/ + // OLSAppTLSHashStateCacheKey is the key of the hash value of the OLS App TLS certificates + OLSAppTLSHashStateCacheKey = "olsapptls-hash" + // OLSConfigHashStateCacheKey is the key of the hash value of the OLSConfig configmap // TelemetryPullSecretNamespace "openshift-config" contains the telemetry pull secret to determine the enablement of telemetry // #nosec G101 TelemetryPullSecretNamespace = "openshift-config" // TelemetryPullSecretName is the name of the secret containing the telemetry pull secret - TelemetryPullSecretName = "pull-secret" + TelemetryPullSecretName = "pull-secret" + OLSDefaultCacheType = "postgres" + PostgresConfigHashStateCacheKey = "olspostgresconfig-hash" + // #nosec G101 + PostgresSecretHashStateCacheKey = "olspostgressecret-hash" ) diff --git a/internal/controller/errors.go b/internal/controller/errors.go index f328b025..ea37f017 100644 --- a/internal/controller/errors.go +++ b/internal/controller/errors.go @@ -14,6 +14,11 @@ const ( ErrCreateSARClusterRoleBinding = "failed to create SAR cluster role binding" ErrCreateServiceMonitor = "failed to create ServiceMonitor" ErrCreatePrometheusRule = "failed to create PrometheusRule" + ErrCreatePostgresSecret = "failed to create OLS Postgres secret" + ErrCreatePostgresBootstrapSecret = "failed to create OLS Postgres bootstrap secret" + ErrCreatePostgresConfigMap = "failed to create OLS Postgres configmap" + ErrCreatePostgresService = "failed to create OLS Postgres service" + ErrCreatePostgresDeployment = "failed to create OLS Postgres deployment" ErrDeleteConsolePlugin = "failed to delete Console Plugin" ErrDeleteAdditionalCACM = "failed to delete additional CA configmap" ErrGenerateAdditionalCACM = "failed to generate additional CA configmap" @@ -26,11 +31,18 @@ const ( ErrGenerateConsolePluginDeployment = "failed to generate Console Plugin deployment" ErrGenerateConsolePluginService = "failed to generate Console Plugin service" ErrGenerateHash = "failed to generate hash for the existing OLS configmap" + ErrGenerateProviderCredentialsHash = "failed to generate OLS provider credentials hash" ErrGenerateSARClusterRole = "failed to generate SAR cluster role" ErrGenerateSARClusterRoleBinding = "failed to generate SAR cluster role binding" ErrGenerateServiceMonitor = "failed to generate ServiceMonitor" ErrGeneratePrometheusRule = "failed to generate PrometheusRule" ErrGetAdditionalCACM = "failed to get additional CA configmap" + ErrGeneratePostgresSecret = "failed to generate OLS Postgres secret" + ErrGeneratePostgresBootstrapSecret = "failed to generate OLS Postgres bootstrap secret" + ErrGeneratePostgresConfigMap = "failed to generate OLS Postgres configmap" + ErrGeneratePostgresService = "failed to generate OLS Postgres service" + ErrGeneratePostgresSecretHash = "failed to generate hash for the existing OLS postgres secret" + ErrGeneratePostgresDeployment = "failed to generate OLS Postgres deployment" ErrGetAPIConfigmap = "failed to get OLS configmap" ErrGetAPIDeployment = "failed to get OLS deployment" ErrGetAPIService = "failed to get OLS service" @@ -58,4 +70,16 @@ const ( ErrUpdateCRStatusCondition = "failed to update OLSConfig CR status condition" ErrUpdateServiceMonitor = "failed to update ServiceMonitor" ErrUpdatePrometheusRule = "failed to update PrometheusRule" + // #nosec G101 + ErrGetPostgresSecret = "failed to get OLS Postgres secret" + // #nosec G101 + ErrGetPostgresBootstrapSecret = "failed to get OLS Postgres bootstrap secret" + ErrGetPostgresConfigMap = "failed to get OLS Postgres configmap" + ErrGetPostgresService = "failed to get OLS Postgres service" + ErrGetPostgresDeployment = "failed to get OLS Postgres deployment" + ErrUpdateProviderSecret = "failed to update provider secret" + ErrUpdatePostgresSecret = "failed to update OLS Postgres secret" + ErrUpdatePostgresDeployment = "failed to update OLS Postgres deployment" + ErrListOldPostgresSecrets = "failed to list old OLS Postgres secrets" + ErrDeleteOldPostgresSecrets = "failed to delete old OLS Postgres secret" ) diff --git a/internal/controller/ols_app_postgres_assets.go b/internal/controller/ols_app_postgres_assets.go new file mode 100644 index 00000000..37d5d859 --- /dev/null +++ b/internal/controller/ols_app_postgres_assets.go @@ -0,0 +1,342 @@ +package controller + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "path" + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" +) + +func generatePostgresSelectorLabels() map[string]string { + return map[string]string{ + "app.kubernetes.io/component": "postgres-server", + "app.kubernetes.io/managed-by": "lightspeed-operator", + "app.kubernetes.io/name": "lightspeed-service-postgres", + "app.kubernetes.io/part-of": "openshift-lightspeed", + } +} + +func getPostgresCAConfigVolume() corev1.Volume { + return corev1.Volume{ + Name: PostgresCAVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: OLSCAConfigMap, + }, + }, + }, + } +} + +func getPostgresCAVolumeMount(mountPath string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: PostgresCAVolume, + MountPath: mountPath, + ReadOnly: true, + } +} + +func (r *OLSConfigReconciler) generatePostgresDeployment(cr *olsv1alpha1.OLSConfig) (*appsv1.Deployment, error) { + cacheReplicas := int32(1) + revisionHistoryLimit := int32(1) + postgresSecretName := PostgresSecretName + if cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret != "" { + postgresSecretName = cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret + } + + passwordMap, err := getSecretContent(r.Client, postgresSecretName, r.Options.Namespace, []string{OLSComponentPasswordFileName}, &corev1.Secret{}) + if err != nil { + return nil, fmt.Errorf("Password is a must to start postgres deployment : %w", err) + } + postgresPassword := passwordMap[OLSComponentPasswordFileName] + if cr.Spec.OLSConfig.ConversationCache.Postgres.SharedBuffers == "" { + cr.Spec.OLSConfig.ConversationCache.Postgres.SharedBuffers = PostgresSharedBuffers + } + if cr.Spec.OLSConfig.ConversationCache.Postgres.MaxConnections == 0 { + cr.Spec.OLSConfig.ConversationCache.Postgres.MaxConnections = PostgresMaxConnections + } + defaultPermission := int32(0600) + tlsCertsVolume := corev1.Volume{ + Name: "secret-" + PostgresCertsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: PostgresCertsSecretName, + DefaultMode: &defaultPermission, + }, + }, + } + bootstrapVolume := corev1.Volume{ + Name: "secret-" + PostgresBootstrapSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: PostgresBootstrapSecretName, + }, + }, + } + configVolume := corev1.Volume{ + Name: PostgresConfigMap, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: PostgresConfigMap}, + }, + }, + } + dataVolume := corev1.Volume{ + Name: PostgresDataVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + volumes := []corev1.Volume{tlsCertsVolume, bootstrapVolume, configVolume, dataVolume, getPostgresCAConfigVolume()} + postgresTLSVolumeMount := corev1.VolumeMount{ + Name: "secret-" + PostgresCertsSecretName, + MountPath: OLSAppCertsMountRoot, + ReadOnly: true, + } + bootstrapVolumeMount := corev1.VolumeMount{ + Name: "secret-" + PostgresBootstrapSecretName, + MountPath: PostgresBootstrapVolumeMountPath, + SubPath: PostgresExtensionScript, + ReadOnly: true, + } + configVolumeMount := corev1.VolumeMount{ + Name: PostgresConfigMap, + MountPath: PostgresConfigVolumeMountPath, + SubPath: PostgresConfig, + } + dataVolumeMount := corev1.VolumeMount{ + Name: PostgresDataVolume, + MountPath: PostgresDataVolumeMountPath, + } + volumeMounts := []corev1.VolumeMount{postgresTLSVolumeMount, bootstrapVolumeMount, configVolumeMount, dataVolumeMount, getPostgresCAVolumeMount(path.Join(OLSAppCertsMountRoot, PostgresCAVolume))} + deployment := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: PostgresDeploymentName, + Namespace: r.Options.Namespace, + Labels: generatePostgresSelectorLabels(), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &cacheReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: generatePostgresSelectorLabels(), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: generatePostgresSelectorLabels(), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: PostgresDeploymentName, + Image: r.Options.LightspeedServicePostgresImage, + ImagePullPolicy: corev1.PullAlways, + Ports: []corev1.ContainerPort{ + { + Name: "server", + ContainerPort: PostgresServicePort, + Protocol: corev1.ProtocolTCP, + }, + }, + VolumeMounts: volumeMounts, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + }, + Env: []corev1.EnvVar{ + { + Name: "POSTGRESQL_USER", + Value: PostgresDefaultUser, + }, + { + Name: "POSTGRESQL_DATABASE", + Value: PostgresDefaultDbName, + }, + { + Name: "POSTGRESQL_ADMIN_PASSWORD", + Value: postgresPassword, + }, + { + Name: "POSTGRESQL_PASSWORD", + Value: postgresPassword, + }, + { + Name: "POSTGRESQL_SHARED_BUFFERS", + Value: cr.Spec.OLSConfig.ConversationCache.Postgres.SharedBuffers, + }, + { + Name: "POSTGRESQL_MAX_CONNECTIONS", + Value: strconv.Itoa(cr.Spec.OLSConfig.ConversationCache.Postgres.MaxConnections), + }, + }, + }, + }, + Volumes: volumes, + }, + }, + RevisionHistoryLimit: &revisionHistoryLimit, + }, + } + + if err := controllerutil.SetControllerReference(cr, &deployment, r.Scheme); err != nil { + return nil, err + } + + return &deployment, nil +} + +// updatePostgresDeployment updates the deployment based on CustomResource configuration. +func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, existingDeployment, desiredDeployment *appsv1.Deployment) error { + changed := false + + // Validate deployment annotations. + if existingDeployment.Annotations == nil || + existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] || + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { + updateDeploymentAnnotations(existingDeployment, map[string]string{ + PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], + }) + // update the deployment template annotation triggers the rolling update + updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ + PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], + }) + + if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil { + return err + } + + changed = true + } + + if changed { + r.logger.Info("updating OLS postgres deployment", "name", existingDeployment.Name) + if err := r.Update(ctx, existingDeployment); err != nil { + return err + } + } else { + r.logger.Info("OLS postgres deployment reconciliation skipped", "deployment", existingDeployment.Name, "olsconfig hash", existingDeployment.Annotations[PostgresConfigHashKey]) + } + + return nil +} + +func (r *OLSConfigReconciler) generatePostgresService(cr *olsv1alpha1.OLSConfig) (*corev1.Service, error) { + service := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: PostgresServiceName, + Namespace: r.Options.Namespace, + Labels: generatePostgresSelectorLabels(), + Annotations: map[string]string{ + ServingCertSecretAnnotationKey: PostgresCertsSecretName, + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Port: PostgresServicePort, + Protocol: corev1.ProtocolTCP, + Name: "server", + TargetPort: intstr.Parse("server"), + }, + }, + Selector: generatePostgresSelectorLabels(), + Type: corev1.ServiceTypeClusterIP, + }, + } + + if err := controllerutil.SetControllerReference(cr, &service, r.Scheme); err != nil { + return nil, err + } + + return &service, nil +} + +func (r *OLSConfigReconciler) generatePostgresSecret(cr *olsv1alpha1.OLSConfig) (*corev1.Secret, error) { + postgresSecretName := PostgresSecretName + if cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret != "" { + postgresSecretName = cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret + } + randomPassword := make([]byte, 12) + _, err := rand.Read(randomPassword) + if err != nil { + return nil, fmt.Errorf("Error generating random password: %w", err) + } + // Encode the password to base64 + encodedPassword := base64.StdEncoding.EncodeToString(randomPassword) + passwordHash, err := hashBytes([]byte(encodedPassword)) + if err != nil { + return nil, fmt.Errorf("failed to generate OLS postgres password hash %w", err) + } + secret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: postgresSecretName, + Namespace: r.Options.Namespace, + Labels: generatePostgresSelectorLabels(), + Annotations: map[string]string{ + PostgresSecretHashKey: passwordHash, + }, + }, + Data: map[string][]byte{ + PostgresSecretKeyName: []byte(encodedPassword), + }, + } + + if err := controllerutil.SetControllerReference(cr, &secret, r.Scheme); err != nil { + return nil, err + } + + return &secret, nil +} + +func (r *OLSConfigReconciler) generatePostgresBootstrapSecret(cr *olsv1alpha1.OLSConfig) (*corev1.Secret, error) { + secret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: PostgresBootstrapSecretName, + Namespace: r.Options.Namespace, + Labels: generatePostgresSelectorLabels(), + }, + StringData: map[string]string{ + PostgresExtensionScript: string(PostgresBootStrapScriptContent), + }, + } + + if err := controllerutil.SetControllerReference(cr, &secret, r.Scheme); err != nil { + return nil, err + } + + return &secret, nil +} + +func (r *OLSConfigReconciler) generatePostgresConfigMap(cr *olsv1alpha1.OLSConfig) (*corev1.ConfigMap, error) { + configMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: PostgresConfigMap, + Namespace: r.Options.Namespace, + Labels: generatePostgresSelectorLabels(), + }, + Data: map[string]string{ + PostgresConfig: PostgresConfigMapContent, + }, + } + + if err := controllerutil.SetControllerReference(cr, &configMap, r.Scheme); err != nil { + return nil, err + } + + return &configMap, nil +} diff --git a/internal/controller/ols_app_postgres_assets_test.go b/internal/controller/ols_app_postgres_assets_test.go new file mode 100644 index 00000000..622b5a01 --- /dev/null +++ b/internal/controller/ols_app_postgres_assets_test.go @@ -0,0 +1,431 @@ +package controller + +import ( + "path" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" +) + +var _ = Describe("App postgres server assets", func() { + + var cr *olsv1alpha1.OLSConfig + var r *OLSConfigReconciler + var rOptions *OLSConfigReconcilerOptions + + validatePostgresDeployment := func(dep *appsv1.Deployment, password string) { + replicas := int32(1) + revisionHistoryLimit := int32(1) + defaultPermission := int32(0600) + Expect(dep.Name).To(Equal(PostgresDeploymentName)) + Expect(dep.Namespace).To(Equal(OLSNamespaceDefault)) + Expect(dep.Spec.Template.Spec.Containers[0].Image).To(Equal(rOptions.LightspeedServicePostgresImage)) + Expect(dep.Spec.Template.Spec.Containers[0].Name).To(Equal("lightspeed-postgres-server")) + Expect(dep.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) + Expect(dep.Spec.Template.Spec.Containers[0].Ports).To(Equal([]corev1.ContainerPort{ + { + ContainerPort: PostgresServicePort, + Name: "server", + Protocol: corev1.ProtocolTCP, + }, + })) + Expect(dep.Spec.Template.Spec.Containers[0].Resources).To(Equal(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + })) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(Equal([]corev1.EnvVar{ + { + Name: "POSTGRESQL_USER", + Value: PostgresDefaultUser, + }, + { + Name: "POSTGRESQL_DATABASE", + Value: PostgresDefaultDbName, + }, + { + Name: "POSTGRESQL_ADMIN_PASSWORD", + Value: password, + }, + { + Name: "POSTGRESQL_PASSWORD", + Value: password, + }, + { + Name: "POSTGRESQL_SHARED_BUFFERS", + Value: PostgresSharedBuffers, + }, + { + Name: "POSTGRESQL_MAX_CONNECTIONS", + Value: strconv.Itoa(PostgresMaxConnections), + }, + })) + Expect(dep.Spec.Selector.MatchLabels).To(Equal(generatePostgresSelectorLabels())) + Expect(dep.Spec.RevisionHistoryLimit).To(Equal(&revisionHistoryLimit)) + Expect(dep.Spec.Replicas).To(Equal(&replicas)) + Expect(dep.Spec.Template.Spec.Containers[0].VolumeMounts).To(Equal([]corev1.VolumeMount{ + { + Name: "secret-" + PostgresCertsSecretName, + MountPath: OLSAppCertsMountRoot, + ReadOnly: true, + }, + { + Name: "secret-" + PostgresBootstrapSecretName, + MountPath: PostgresBootstrapVolumeMountPath, + SubPath: PostgresExtensionScript, + ReadOnly: true, + }, + { + Name: PostgresConfigMap, + MountPath: PostgresConfigVolumeMountPath, + SubPath: PostgresConfig, + }, + { + Name: PostgresDataVolume, + MountPath: PostgresDataVolumeMountPath, + }, + { + Name: PostgresCAVolume, + MountPath: path.Join(OLSAppCertsMountRoot, PostgresCAVolume), + ReadOnly: true, + }, + })) + Expect(dep.Spec.Template.Spec.Volumes).To(Equal([]corev1.Volume{ + { + Name: "secret-" + PostgresCertsSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: PostgresCertsSecretName, + DefaultMode: &defaultPermission, + }, + }, + }, + { + Name: "secret-" + PostgresBootstrapSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: PostgresBootstrapSecretName, + }, + }, + }, + { + Name: PostgresConfigMap, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: PostgresConfigMap}, + }, + }, + }, + { + Name: PostgresDataVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: PostgresCAVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: OLSCAConfigMap}, + }, + }, + }, + })) + } + + validatePostgresService := func(service *corev1.Service, err error) { + Expect(err).NotTo(HaveOccurred()) + Expect(service.Name).To(Equal(PostgresServiceName)) + Expect(service.Namespace).To(Equal(OLSNamespaceDefault)) + Expect(service.Labels).To(Equal(generatePostgresSelectorLabels())) + Expect(service.Annotations).To(Equal(map[string]string{ + "service.beta.openshift.io/serving-cert-secret-name": PostgresCertsSecretName, + })) + Expect(service.Spec.Selector).To(Equal(generatePostgresSelectorLabels())) + Expect(service.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(service.Spec.Ports).To(Equal([]corev1.ServicePort{ + { + Name: "server", + Port: PostgresServicePort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.Parse("server"), + }, + })) + } + + validatePostgresConfigMap := func(configMap *corev1.ConfigMap) { + Expect(configMap.Namespace).To(Equal(cr.Namespace)) + Expect(configMap.Labels).To(Equal(generatePostgresSelectorLabels())) + Expect(configMap.Data).To(HaveKey(PostgresConfig)) + } + + validatePostgresSecret := func(secret *corev1.Secret) { + Expect(secret.Namespace).To(Equal(cr.Namespace)) + Expect(secret.Labels).To(Equal(generatePostgresSelectorLabels())) + Expect(secret.Annotations).To(HaveKey(PostgresSecretHashKey)) + Expect(secret.Data).To(HaveKey(PostgresSecretKeyName)) + } + + validatePostgresBootstrapSecret := func(secret *corev1.Secret) { + Expect(secret.Namespace).To(Equal(cr.Namespace)) + Expect(secret.Labels).To(Equal(generatePostgresSelectorLabels())) + Expect(secret.StringData).To(HaveKey(PostgresExtensionScript)) + } + + Context("complete custom resource", func() { + BeforeEach(func() { + rOptions = &OLSConfigReconcilerOptions{ + LightspeedServicePostgresImage: "lightspeed-service-postgres:latest", + Namespace: OLSNamespaceDefault, + } + cr = getOLSConfigWithCacheCR() + r = &OLSConfigReconciler{ + Options: *rOptions, + logger: logf.Log.WithName("olsconfig.reconciler"), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + stateCache: make(map[string]string), + } + }) + + It("should generate the OLS postgres deployment", func() { + cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = "dummy-secret-1" + secret, _ := r.generatePostgresSecret(cr) + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: "dummy-secret-1", + }, + }) + secretCreationErr := r.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + passwordMap, _ := getSecretContent(r.Client, secret.Name, cr.Namespace, []string{OLSComponentPasswordFileName}, secret) + password := passwordMap[OLSComponentPasswordFileName] + deployment, err := r.generatePostgresDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + validatePostgresDeployment(deployment, password) + secretDeletionErr := r.Delete(ctx, secret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + }) + + It("should work when no update in the OLS postgres deployment", func() { + cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = "dummy-secret-2" + secret, _ := r.generatePostgresSecret(cr) + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: "dummy-secret-2", + }, + }) + secretCreationErr := r.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + deployment, err := r.generatePostgresDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + deployment.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Deployment", + APIVersion: "apps/v1", + UID: "ownerUID", + Name: "lightspeed-postgres-server-1", + }, + }) + deployment.ObjectMeta.Name = "lightspeed-postgres-server-1" + deploymentCreationErr := r.Create(ctx, deployment) + Expect(deploymentCreationErr).NotTo(HaveOccurred()) + updateErr := r.updatePostgresDeployment(ctx, deployment, deployment) + Expect(updateErr).NotTo(HaveOccurred()) + secretDeletionErr := r.Delete(ctx, secret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + deploymentDeletionErr := r.Delete(ctx, deployment) + Expect(deploymentDeletionErr).NotTo(HaveOccurred()) + }) + + It("should work when there is an update in the OLS postgres deployment", func() { + cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = "dummy-secret-3" + secret, _ := r.generatePostgresSecret(cr) + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: "dummy-secret-3", + }, + }) + secretCreationErr := r.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + deployment, err := r.generatePostgresDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + deployment.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Deployment", + APIVersion: "apps/v1", + UID: "ownerUID", + Name: "lightspeed-postgres-server-2", + }, + }) + deployment.ObjectMeta.Name = "lightspeed-postgres-server-2" + deploymentCreationErr := r.Create(ctx, deployment) + Expect(deploymentCreationErr).NotTo(HaveOccurred()) + deploymentClone := deployment.DeepCopy() + deploymentClone.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{ + { + Name: "DUMMY_UPDATE", + Value: PostgresDefaultUser, + }, + } + updateErr := r.updatePostgresDeployment(ctx, deployment, deploymentClone) + Expect(updateErr).NotTo(HaveOccurred()) + Expect(deployment.Spec.Template.Spec.Containers[0].Env).To(Equal([]corev1.EnvVar{ + { + Name: "DUMMY_UPDATE", + Value: PostgresDefaultUser, + }, + })) + secretDeletionErr := r.Delete(ctx, secret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + deploymentDeletionErr := r.Delete(ctx, deployment) + Expect(deploymentDeletionErr).NotTo(HaveOccurred()) + }) + + It("should generate the OLS postgres service", func() { + validatePostgresService(r.generatePostgresService(cr)) + }) + + It("should generate the OLS postgres configmap", func() { + configMap, err := r.generatePostgresConfigMap(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(configMap.Name).To(Equal(PostgresConfigMap)) + validatePostgresConfigMap(configMap) + }) + + It("should generate the OLS postgres secret", func() { + secret, err := r.generatePostgresSecret(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Name).To(Equal("lightspeed-postgres-secret")) + validatePostgresSecret(secret) + }) + + It("should generate the OLS postgres bootstrap secret", func() { + secret, err := r.generatePostgresBootstrapSecret(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Name).To(Equal(PostgresBootstrapSecretName)) + validatePostgresBootstrapSecret(secret) + }) + }) + + Context("empty custom resource", func() { + BeforeEach(func() { + rOptions = &OLSConfigReconcilerOptions{ + LightspeedServicePostgresImage: "lightspeed-service-postgres:latest", + Namespace: OLSNamespaceDefault, + } + cr = getNoCacheCR() + r = &OLSConfigReconciler{ + Options: *rOptions, + logger: logf.Log.WithName("olsconfig.reconciler"), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + stateCache: make(map[string]string), + } + }) + + It("should generate the OLS postgres deployment", func() { + cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = "dummy-secret-4" + cr.Spec.OLSConfig.ConversationCache.Postgres.User = PostgresDefaultUser + cr.Spec.OLSConfig.ConversationCache.Postgres.DbName = PostgresDefaultDbName + cr.Spec.OLSConfig.ConversationCache.Postgres.SharedBuffers = PostgresSharedBuffers + cr.Spec.OLSConfig.ConversationCache.Postgres.MaxConnections = PostgresMaxConnections + secret, _ := r.generatePostgresSecret(cr) + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: "dummy-secret-4", + }, + }) + secretCreationErr := r.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + passwordMap, _ := getSecretContent(r.Client, secret.Name, cr.Namespace, []string{OLSComponentPasswordFileName}, secret) + password := passwordMap[OLSComponentPasswordFileName] + deployment, err := r.generatePostgresDeployment(cr) + Expect(err).NotTo(HaveOccurred()) + validatePostgresDeployment(deployment, password) + secretDeletionErr := r.Delete(ctx, secret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + }) + + It("should generate the OLS postgres service", func() { + validatePostgresService(r.generatePostgresService(cr)) + }) + + It("should generate the OLS postgres configmap", func() { + configMap, err := r.generatePostgresConfigMap(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(configMap.Name).To(Equal(PostgresConfigMap)) + validatePostgresConfigMap(configMap) + }) + + It("should generate the OLS postgres bootstrap secret", func() { + secret, err := r.generatePostgresBootstrapSecret(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Name).To(Equal(PostgresBootstrapSecretName)) + validatePostgresBootstrapSecret(secret) + }) + + It("should generate the OLS postgres secret", func() { + cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = PostgresSecretName + secret, err := r.generatePostgresSecret(cr) + Expect(err).NotTo(HaveOccurred()) + Expect(secret.Name).To(Equal("lightspeed-postgres-secret")) + validatePostgresSecret(secret) + }) + }) + +}) + +func getOLSConfigWithCacheCR() *olsv1alpha1.OLSConfig { + return &olsv1alpha1.OLSConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: OLSNamespaceDefault, + }, + Spec: olsv1alpha1.OLSConfigSpec{ + OLSConfig: olsv1alpha1.OLSSpec{ + ConversationCache: olsv1alpha1.ConversationCacheSpec{ + Type: olsv1alpha1.Postgres, + Postgres: olsv1alpha1.PostgresSpec{ + User: PostgresDefaultUser, + DbName: PostgresDefaultDbName, + SharedBuffers: PostgresSharedBuffers, + MaxConnections: PostgresMaxConnections, + CredentialsSecret: PostgresSecretName, + }, + }, + }, + }, + } +} + +func getNoCacheCR() *olsv1alpha1.OLSConfig { + return &olsv1alpha1.OLSConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Namespace: OLSNamespaceDefault, + }, + } +} diff --git a/internal/controller/ols_app_postgres_reconciliator.go b/internal/controller/ols_app_postgres_reconciliator.go new file mode 100644 index 00000000..685e2c19 --- /dev/null +++ b/internal/controller/ols_app_postgres_reconciliator.go @@ -0,0 +1,213 @@ +package controller + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" +) + +func (r *OLSConfigReconciler) reconcilePostgresServer(ctx context.Context, olsconfig *olsv1alpha1.OLSConfig) error { + r.logger.Info("reconcilePostgresServer starts") + tasks := []ReconcileTask{ + { + Name: "reconcile Postgres ConfigMap", + Task: r.reconcilePostgresConfigMap, + }, + { + Name: "reconcile Postgres Bootstrap Secret", + Task: r.reconcilePostgresBootstrapSecret, + }, + { + Name: "reconcile Postgres Secret", + Task: r.reconcilePostgresSecret, + }, + { + Name: "reconcile Postgres Service", + Task: r.reconcilePostgresService, + }, + { + Name: "reconcile Postgres Deployment", + Task: r.reconcilePostgresDeployment, + }, + } + + for _, task := range tasks { + err := task.Task(ctx, olsconfig) + if err != nil { + r.logger.Error(err, "reconcilePostgresServer error", "task", task.Name) + return fmt.Errorf("failed to %s: %w", task.Name, err) + } + } + + r.logger.Info("reconcilePostgresServer completed") + + return nil +} + +func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + desiredDeployment, err := r.generatePostgresDeployment(cr) + if err != nil { + return fmt.Errorf("%s: %w", ErrGeneratePostgresDeployment, err) + } + + existingDeployment := &appsv1.Deployment{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) + if err != nil && errors.IsNotFound(err) { + updateDeploymentAnnotations(desiredDeployment, map[string]string{ + PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], + }) + updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ + PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], + }) + r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name) + err = r.Create(ctx, desiredDeployment) + if err != nil { + return fmt.Errorf("%s: %w", ErrCreatePostgresDeployment, err) + } + return nil + } else if err != nil { + return fmt.Errorf("%s: %w", ErrGetPostgresDeployment, err) + } + + err = r.updatePostgresDeployment(ctx, existingDeployment, desiredDeployment) + + if err != nil { + return fmt.Errorf("%s: %w", ErrUpdatePostgresDeployment, err) + } + + r.logger.Info("OLS postgres deployment reconciled", "deployment", desiredDeployment.Name) + return nil +} + +func (r *OLSConfigReconciler) reconcilePostgresService(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + service, err := r.generatePostgresService(cr) + if err != nil { + return fmt.Errorf("%s: %w", ErrGeneratePostgresService, err) + } + + foundService := &corev1.Service{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresServiceName, Namespace: r.Options.Namespace}, foundService) + if err != nil && errors.IsNotFound(err) { + err = r.Create(ctx, service) + if err != nil { + return fmt.Errorf("%s: %w", ErrCreatePostgresService, err) + } + } else if err != nil { + return fmt.Errorf("%s: %w", ErrGetPostgresService, err) + } + r.logger.Info("OLS postgres service reconciled", "service", service.Name) + return nil +} + +func (r *OLSConfigReconciler) reconcilePostgresConfigMap(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + configMap, err := r.generatePostgresConfigMap(cr) + if err != nil { + return fmt.Errorf("%s: %w", ErrGeneratePostgresConfigMap, err) + } + + foundConfigMap := &corev1.ConfigMap{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresConfigMap, Namespace: r.Options.Namespace}, foundConfigMap) + if err != nil && errors.IsNotFound(err) { + err = r.Create(ctx, configMap) + if err != nil { + return fmt.Errorf("%s: %w", ErrCreatePostgresConfigMap, err) + } + } else if err != nil { + return fmt.Errorf("%s: %w", ErrGetPostgresConfigMap, err) + } + r.logger.Info("OLS postgres configmap reconciled", "configmap", configMap.Name) + return nil +} + +func (r *OLSConfigReconciler) reconcilePostgresBootstrapSecret(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + secret, err := r.generatePostgresBootstrapSecret(cr) + if err != nil { + return fmt.Errorf("%s: %w", ErrGeneratePostgresBootstrapSecret, err) + } + + foundSecret := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresBootstrapSecretName, Namespace: r.Options.Namespace}, foundSecret) + if err != nil && errors.IsNotFound(err) { + err = r.Create(ctx, secret) + if err != nil { + return fmt.Errorf("%s: %w", ErrCreatePostgresBootstrapSecret, err) + } + } else if err != nil { + return fmt.Errorf("%s: %w", ErrGetPostgresBootstrapSecret, err) + } + r.logger.Info("OLS postgres bootstrap secret reconciled", "secret", secret.Name) + return nil +} + +func (r *OLSConfigReconciler) reconcilePostgresSecret(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + secret, err := r.generatePostgresSecret(cr) + if err != nil { + return fmt.Errorf("%s: %w", ErrGeneratePostgresSecret, err) + } + foundSecret := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{Name: secret.Name, Namespace: r.Options.Namespace}, foundSecret) + if err != nil && errors.IsNotFound(err) { + err = r.deleteOldPostgresSecrets(ctx) + if err != nil { + return err + } + r.logger.Info("creating a new Postgres secret", "secret", secret.Name) + err = r.Create(ctx, secret) + if err != nil { + return fmt.Errorf("%s: %w", ErrCreatePostgresSecret, err) + } + r.stateCache[PostgresSecretHashStateCacheKey] = secret.Annotations[PostgresSecretHashKey] + return nil + } else if err != nil { + return fmt.Errorf("%s: %w", ErrGetPostgresSecret, err) + } + foundSecretHash, err := hashBytes(foundSecret.Data[PostgresSecretKeyName]) + if err != nil { + return fmt.Errorf("%s: %w", ErrGeneratePostgresSecretHash, err) + } + if foundSecretHash == r.stateCache[PostgresSecretHashStateCacheKey] { + r.logger.Info("OLS postgres secret reconciliation skipped", "secret", foundSecret.Name, "hash", foundSecret.Annotations[PostgresSecretHashKey]) + return nil + } + r.stateCache[PostgresSecretHashStateCacheKey] = foundSecretHash + secret.Annotations[PostgresSecretHashKey] = foundSecretHash + secret.Data[PostgresSecretKeyName] = foundSecret.Data[PostgresSecretKeyName] + err = r.Update(ctx, secret) + if err != nil { + return fmt.Errorf("%s: %w", ErrUpdatePostgresSecret, err) + } + r.logger.Info("OLS postgres reconciled", "secret", secret.Name, "hash", secret.Annotations[PostgresSecretHashKey]) + return nil +} + +func (r *OLSConfigReconciler) deleteOldPostgresSecrets(ctx context.Context) error { + labelSelector := labels.Set{"app.kubernetes.io/name": "lightspeed-service-postgres"}.AsSelector() + matchingLabels := client.MatchingLabelsSelector{Selector: labelSelector} + oldSecrets := &corev1.SecretList{} + err := r.Client.List(ctx, oldSecrets, &client.ListOptions{Namespace: r.Options.Namespace, LabelSelector: labelSelector}) + if err != nil { + return fmt.Errorf("failed to list old Postgres secrets: %w", err) + } + r.logger.Info("deleting old Postgres secrets", "count", len(oldSecrets.Items)) + + deleteOptions := &client.DeleteAllOfOptions{ + ListOptions: client.ListOptions{ + Namespace: r.Options.Namespace, + LabelSelector: matchingLabels, + }, + } + if err := r.Client.DeleteAllOf(ctx, &corev1.Secret{}, deleteOptions); err != nil { + return fmt.Errorf("failed to delete old Postgres secrets: %w", err) + } + return nil +} diff --git a/internal/controller/ols_app_postgres_reconciliator_test.go b/internal/controller/ols_app_postgres_reconciliator_test.go new file mode 100644 index 00000000..1844ac43 --- /dev/null +++ b/internal/controller/ols_app_postgres_reconciliator_test.go @@ -0,0 +1,169 @@ +package controller + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("Postgres server reconciliator", Ordered, func() { + + Context("Creation logic", Ordered, func() { + var secret *corev1.Secret + var bootstrapSecret *corev1.Secret + BeforeEach(func() { + By("create the provider secret") + secret, _ = generateRandomSecret() + secret.Name = "lightspeed-postgres-secret" + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID1", + Name: "lightspeed-postgres-secret", + }, + }) + secretCreationErr := reconciler.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("create the tls secret") + tlsSecret, _ = generateRandomSecret() + tlsSecret.Name = OLSCertsSecretName + tlsSecret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID", + Name: OLSCertsSecretName, + }, + }) + secretCreationErr = reconciler.Create(ctx, tlsSecret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("create the bootstrap secret") + bootstrapSecret, _ = generateRandomSecret() + bootstrapSecret.Name = "lightspeed-bootstrap-secret" + bootstrapSecret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID2", + Name: "lightspeed-bootstrap-secret", + }, + }) + bootstrapSecretCreationErr := reconciler.Create(ctx, bootstrapSecret) + Expect(bootstrapSecretCreationErr).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + By("Delete the provider secret") + secretDeletionErr := reconciler.Delete(ctx, secret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + + By("Delete the tls secret") + secretDeletionErr = reconciler.Delete(ctx, tlsSecret) + Expect(secretDeletionErr).NotTo(HaveOccurred()) + + By("Delete the bootstrap secret") + bootstrapSecretDeletionErr := reconciler.Delete(ctx, bootstrapSecret) + Expect(bootstrapSecretDeletionErr).NotTo(HaveOccurred()) + }) + + It("should reconcile from OLSConfig custom resource", func() { + By("Reconcile the OLSConfig custom resource") + cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = PostgresSecretName + err := reconciler.reconcilePostgresServer(ctx, cr) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a service for lightspeed-postgres-server", func() { + + By("Get postgres service") + svc := &corev1.Service{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: PostgresServiceName, Namespace: OLSNamespaceDefault}, svc) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a deployment lightspeed-postgres-server", func() { + + By("Get postgres deployment") + dep := &appsv1.Deployment{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: PostgresDeploymentName, Namespace: OLSNamespaceDefault}, dep) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a postgres configmap", func() { + + By("Get the postgres config") + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: PostgresConfigMap, Namespace: OLSNamespaceDefault}, configMap) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a postgres bootstrap secret", func() { + + By("Get the bootstrap secret") + secret := &corev1.Secret{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: PostgresBootstrapSecretName, Namespace: OLSNamespaceDefault}, secret) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should create a postgres secret", func() { + + By("Get the postgres secret") + secret := &corev1.Secret{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: PostgresSecretName, Namespace: OLSNamespaceDefault}, secret) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should trigger a rolling deployment when there is an update in secret name", func() { + + By("create the test secret") + secret, _ = generateRandomSecret() + secret.SetOwnerReferences([]metav1.OwnerReference{ + { + Kind: "Secret", + APIVersion: "v1", + UID: "ownerUID1", + Name: "test-secret", + }, + }) + secretCreationErr := reconciler.Create(ctx, secret) + Expect(secretCreationErr).NotTo(HaveOccurred()) + + By("Get the postgres deployment") + dep := &appsv1.Deployment{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: PostgresDeploymentName, Namespace: OLSNamespaceDefault}, dep) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Spec.Template.Annotations).NotTo(BeNil()) + oldHash := dep.Spec.Template.Annotations[PostgresConfigHashKey] + + By("Update the OLSConfig custom resource") + olsConfig := &olsv1alpha1.OLSConfig{} + err = k8sClient.Get(ctx, crNamespacedName, olsConfig) + Expect(err).NotTo(HaveOccurred()) + olsConfig.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret = "dummy-secret" + + By("Reconcile the app server") + err = reconciler.reconcileAppServer(ctx, olsConfig) + Expect(err).NotTo(HaveOccurred()) + By("Reconcile the postgres server") + err = reconciler.reconcilePostgresServer(ctx, olsConfig) + Expect(err).NotTo(HaveOccurred()) + + By("Get the postgres deployment") + err = k8sClient.Get(ctx, types.NamespacedName{Name: PostgresDeploymentName, Namespace: OLSNamespaceDefault}, dep) + fmt.Printf("%v", dep) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Spec.Template.Annotations).NotTo(BeNil()) + Expect(dep.Annotations[PostgresConfigHashKey]).NotTo(Equal(oldHash)) + }) + }) +}) diff --git a/internal/controller/ols_app_server_assets.go b/internal/controller/ols_app_server_assets.go index f0c823ad..83df860d 100644 --- a/internal/controller/ols_app_server_assets.go +++ b/internal/controller/ols_app_server_assets.go @@ -147,37 +147,23 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv } providerConfigs = append(providerConfigs, providerConfig) } - // TODO: Update DB - // redisMaxMemory := intstr.FromString(RedisMaxMemory) - // redisMaxMemoryPolicy := RedisMaxMemoryPolicy - // redisSecretName := RedisSecretName - // redisConfig := cr.Spec.OLSConfig.ConversationCache.Redis - // if redisConfig.MaxMemory != nil && redisConfig.MaxMemory.String() != "" { - // redisMaxMemory = *cr.Spec.OLSConfig.ConversationCache.Redis.MaxMemory - // } - // if redisConfig.MaxMemoryPolicy != "" { - // redisMaxMemoryPolicy = cr.Spec.OLSConfig.ConversationCache.Redis.MaxMemoryPolicy - // } - // if redisConfig.CredentialsSecret != "" { - // redisSecretName = cr.Spec.OLSConfig.ConversationCache.Redis.CredentialsSecret - // } - // redisPasswordPath := path.Join(CredentialsMountRoot, redisSecretName, OLSComponentPasswordFileName) - // conversationCache := ConversationCacheConfig{ - // Type: string(OLSDefaultCacheType), - // Redis: RedisCacheConfig{ - // Host: strings.Join([]string{RedisServiceName, r.Options.Namespace, "svc"}, "."), - // Port: RedisServicePort, - // MaxMemory: &redisMaxMemory, - // MaxMemoryPolicy: redisMaxMemoryPolicy, - // PasswordPath: redisPasswordPath, - // CACertPath: path.Join(OLSAppCertsMountRoot, RedisCertsSecretName, RedisCAVolume, "service-ca.crt"), - // }, - // } + postgresSecretName := PostgresSecretName + postgresConfig := cr.Spec.OLSConfig.ConversationCache.Postgres + if postgresConfig.CredentialsSecret != "" { + postgresSecretName = cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret + } + postgresPasswordPath := path.Join(CredentialsMountRoot, postgresSecretName, OLSComponentPasswordFileName) conversationCache := ConversationCacheConfig{ - Type: "memory", - Memory: MemoryCacheConfig{ - MaxEntries: 1000, + Type: string(OLSDefaultCacheType), + Postgres: PostgresCacheConfig{ + Host: strings.Join([]string{PostgresServiceName, r.Options.Namespace, "svc"}, "."), + Port: PostgresServicePort, + User: PostgresDefaultUser, + DbName: PostgresDefaultDbName, + PasswordPath: postgresPasswordPath, + SSLMode: PostgresDefaultSSLMode, + CACertPath: path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume, "service-ca.crt"), }, } @@ -263,21 +249,21 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv if err != nil { return nil, fmt.Errorf("failed to generate OLS config file %w", err) } - // TODO: Update DB - // redisConfigFileBytes, err := yaml.Marshal(conversationCache.Redis) - // if err != nil { - // return nil, fmt.Errorf("failed to generate OLS redis config bytes %w", err) - // } + + postgresConfigFileBytes, err := yaml.Marshal(conversationCache.Postgres) + if err != nil { + return nil, fmt.Errorf("failed to generate OLS postgres config bytes %w", err) + } configFileHash, err := hashBytes(configFileBytes) if err != nil { return nil, fmt.Errorf("failed to generate OLS config file hash %w", err) } - // TODO: Update DB - // redisConfigHash, err := hashBytes(redisConfigFileBytes) - // if err != nil { - // return nil, fmt.Errorf("failed to generate OLS redis config hash %w", err) - // } + + postgresConfigHash, err := hashBytes(postgresConfigFileBytes) + if err != nil { + return nil, fmt.Errorf("failed to generate OLS postgres config hash %w", err) + } cm := corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -285,9 +271,8 @@ func (r *OLSConfigReconciler) generateOLSConfigMap(ctx context.Context, cr *olsv Namespace: r.Options.Namespace, Labels: generateAppServerSelectorLabels(), Annotations: map[string]string{ - OLSConfigHashKey: configFileHash, - // TODO: Update DB - //RedisConfigHashKey: redisConfigHash, + OLSConfigHashKey: configFileHash, + PostgresConfigHashKey: postgresConfigHash, }, }, Data: map[string]string{ diff --git a/internal/controller/ols_app_server_assets_test.go b/internal/controller/ols_app_server_assets_test.go index 85e6b580..e16e03f2 100644 --- a/internal/controller/ols_app_server_assets_test.go +++ b/internal/controller/ols_app_server_assets_test.go @@ -10,6 +10,7 @@ import ( "encoding/pem" "math/big" "path" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -84,8 +85,6 @@ var _ = Describe("App server assets", func() { Expect(err).NotTo(HaveOccurred()) cm, err := r.generateOLSConfigMap(context.TODO(), cr) - // TODO: Update DB - //OLSRedisMaxMemory := intstr.FromString(RedisMaxMemory) Expect(err).NotTo(HaveOccurred()) Expect(cm.Name).To(Equal(OLSConfigCmName)) Expect(cm.Namespace).To(Equal(OLSNamespaceDefault)) @@ -101,22 +100,16 @@ var _ = Describe("App server assets", func() { LibLogLevel: "INFO", UvicornLogLevel: "INFO", }, - // TODO: Update DB - // ConversationCache: ConversationCacheConfig{ - // Type: "redis", - // Redis: RedisCacheConfig{ - // Host: strings.Join([]string{RedisServiceName, OLSNamespaceDefault, "svc"}, "."), - // Port: RedisServicePort, - // MaxMemory: &OLSRedisMaxMemory, - // MaxMemoryPolicy: RedisMaxMemoryPolicy, - // PasswordPath: path.Join(CredentialsMountRoot, RedisSecretName, OLSComponentPasswordFileName), - // CACertPath: path.Join(OLSAppCertsMountRoot, RedisCertsSecretName, RedisCAVolume, "service-ca.crt"), - // }, - // }, ConversationCache: ConversationCacheConfig{ - Type: "memory", - Memory: MemoryCacheConfig{ - MaxEntries: 1000, + Type: "postgres", + Postgres: PostgresCacheConfig{ + Host: strings.Join([]string{PostgresServiceName, OLSNamespaceDefault, "svc"}, "."), + Port: PostgresServicePort, + User: PostgresDefaultUser, + DbName: PostgresDefaultDbName, + PasswordPath: path.Join(CredentialsMountRoot, PostgresSecretName, OLSComponentPasswordFileName), + SSLMode: PostgresDefaultSSLMode, + CACertPath: path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume, "service-ca.crt"), }, }, TLSConfig: TLSConfig{ @@ -292,6 +285,16 @@ var _ = Describe("App server assets", func() { ReadOnly: false, MountPath: "/app-root/ols-user-data", }, + { + Name: "secret-lightspeed-postgres-secret", + ReadOnly: true, + MountPath: "/etc/credentials/lightspeed-postgres-secret", + }, + { + Name: "cm-olspostgresca", + ReadOnly: true, + MountPath: path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume), + }, })) Expect(dep.Spec.Template.Spec.Containers[0].Resources).To(Equal(corev1.ResourceRequirements{ Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, @@ -310,15 +313,35 @@ var _ = Describe("App server assets", func() { }, })) Expect(dep.Spec.Template.Spec.Containers[1].VolumeMounts).To(ConsistOf([]corev1.VolumeMount{ + { + Name: "secret-test-secret", + MountPath: path.Join(APIKeyMountRoot, "test-secret"), + ReadOnly: true, + }, + { + Name: "secret-lightspeed-tls", + MountPath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName), + ReadOnly: true, + }, + { + Name: "cm-olsconfig", + MountPath: "/etc/ols", + ReadOnly: true, + }, { Name: "ols-user-data", ReadOnly: false, MountPath: "/app-root/ols-user-data", }, { - Name: "cm-olsconfig", - MountPath: "/etc/ols", + Name: "secret-lightspeed-postgres-secret", ReadOnly: true, + MountPath: "/etc/credentials/lightspeed-postgres-secret", + }, + { + Name: "cm-olspostgresca", + ReadOnly: true, + MountPath: path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume), }, })) Expect(dep.Spec.Template.Spec.Containers[1].Resources).To(Equal(corev1.ResourceRequirements{ @@ -345,6 +368,23 @@ var _ = Describe("App server assets", func() { }, }, }, + { + Name: "secret-lightspeed-postgres-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: PostgresSecretName, + DefaultMode: &defaultVolumeMode, + }, + }, + }, + { + Name: "cm-olspostgresca", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: OLSCAConfigMap}, + }, + }, + }, { Name: "cm-olsconfig", VolumeSource: corev1.VolumeSource{ @@ -403,6 +443,16 @@ var _ = Describe("App server assets", func() { MountPath: "/etc/ols", ReadOnly: true, }, + { + Name: "secret-lightspeed-postgres-secret", + ReadOnly: true, + MountPath: "/etc/credentials/lightspeed-postgres-secret", + }, + { + Name: PostgresCAVolume, + ReadOnly: true, + MountPath: "/etc/certs/lightspeed-postgres-certs/cm-olspostgresca", + }, })) Expect(dep.Spec.Template.Spec.Volumes).To(ConsistOf([]corev1.Volume{ { @@ -432,6 +482,23 @@ var _ = Describe("App server assets", func() { }, }, }, + { + Name: "secret-lightspeed-postgres-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lightspeed-postgres-secret", + DefaultMode: &defaultVolumeMode, + }, + }, + }, + { + Name: PostgresCAVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: OLSCAConfigMap}, + }, + }, + }, })) By("generate deployment without data collector when telemetry pull secret does not contain telemetry token") @@ -475,6 +542,16 @@ var _ = Describe("App server assets", func() { MountPath: "/etc/ols", ReadOnly: true, }, + { + Name: "secret-lightspeed-postgres-secret", + MountPath: "/etc/credentials/lightspeed-postgres-secret", + ReadOnly: true, + }, + { + Name: "cm-olspostgresca", + MountPath: "/etc/certs/lightspeed-postgres-certs/cm-olspostgresca", + ReadOnly: true, + }, })) Expect(dep.Spec.Template.Spec.Volumes).To(ConsistOf([]corev1.Volume{ { @@ -504,6 +581,23 @@ var _ = Describe("App server assets", func() { }, }, }, + { + Name: "secret-lightspeed-postgres-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "lightspeed-postgres-secret", + DefaultMode: &defaultVolumeMode, + }, + }, + }, + { + Name: PostgresCAVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: OLSCAConfigMap}, + }, + }, + }, })) deleteTelemetryPullSecret() }) @@ -578,15 +672,35 @@ var _ = Describe("App server assets", func() { }, })) Expect(deployment.Spec.Template.Spec.Containers[1].VolumeMounts).To(ConsistOf([]corev1.VolumeMount{ + { + Name: "secret-test-secret", + MountPath: path.Join(APIKeyMountRoot, "test-secret"), + ReadOnly: true, + }, + { + Name: "secret-lightspeed-tls", + MountPath: path.Join(OLSAppCertsMountRoot, OLSCertsSecretName), + ReadOnly: true, + }, + { + Name: "cm-olsconfig", + MountPath: "/etc/ols", + ReadOnly: true, + }, { Name: "ols-user-data", ReadOnly: false, MountPath: "/app-root/ols-user-data", }, { - Name: "cm-olsconfig", - MountPath: "/etc/ols", + Name: "secret-lightspeed-postgres-secret", ReadOnly: true, + MountPath: "/etc/credentials/lightspeed-postgres-secret", + }, + { + Name: "cm-olspostgresca", + ReadOnly: true, + MountPath: path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume), }, })) Expect(deployment.Spec.Template.Spec.Volumes).To(ContainElement( @@ -674,9 +788,15 @@ var _ = Describe("App server assets", func() { expectedConfigStr := `llm_providers: [] ols_config: conversation_cache: - memory: - max_entries: 1000 - type: memory + postgres: + ca_cert_path: /etc/certs/lightspeed-postgres-certs/cm-olspostgresca/service-ca.crt + dbname: postgres + host: lightspeed-postgres-server.openshift-lightspeed.svc + password_path: /etc/credentials/lightspeed-postgres-secret/password + port: 5432 + ssl_mode: require + user: postgres + type: postgres logging_config: app_log_level: "" lib_log_level: "" @@ -711,7 +831,7 @@ user_data_collector_config: }) It("should generate the olsconfig config map without user_data_collector_config", func() { - // pull-secret with out telemetry token should make the datacollection disabled + // pull-secret without telemetry token should disable data collection // and user_data_collector_config should not be present in the config createTelemetryPullSecretWithoutTelemetryToken() major, minor, err := r.getClusterVersion(ctx) @@ -723,9 +843,15 @@ user_data_collector_config: expectedConfigStr := `llm_providers: [] ols_config: conversation_cache: - memory: - max_entries: 1000 - type: memory + postgres: + ca_cert_path: /etc/certs/lightspeed-postgres-certs/cm-olspostgresca/service-ca.crt + dbname: postgres + host: lightspeed-postgres-server.openshift-lightspeed.svc + password_path: /etc/credentials/lightspeed-postgres-secret/password + port: 5432 + ssl_mode: require + user: postgres + type: postgres logging_config: app_log_level: "" lib_log_level: "" @@ -814,6 +940,16 @@ user_data_collector_config: {} ReadOnly: false, MountPath: "/app-root/ols-user-data", }, + { + Name: "secret-lightspeed-postgres-secret", + ReadOnly: true, + MountPath: "/etc/credentials/lightspeed-postgres-secret", + }, + { + Name: "cm-olspostgresca", + ReadOnly: true, + MountPath: path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume), + }, })) Expect(dep.Spec.Template.Spec.Volumes).To(ConsistOf([]corev1.Volume{ { @@ -834,6 +970,23 @@ user_data_collector_config: {} }, }, }, + { + Name: "secret-lightspeed-postgres-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: PostgresSecretName, + DefaultMode: &defaultVolumeMode, + }, + }, + }, + { + Name: "cm-olspostgresca", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: OLSCAConfigMap}, + }, + }, + }, { Name: "ols-user-data", VolumeSource: corev1.VolumeSource{ @@ -1090,10 +1243,12 @@ func generateRandomSecret() (*corev1.Secret, error) { secret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", - Namespace: "openshift-lightspeed", - Labels: generateAppServerSelectorLabels(), - Annotations: map[string]string{}, + Name: "test-secret", + Namespace: "openshift-lightspeed", + Labels: generateAppServerSelectorLabels(), + Annotations: map[string]string{ + PostgresSecretHashKey: "test-hash", + }, }, Data: map[string][]byte{ "client_secret": []byte(passwordHash), @@ -1151,6 +1306,12 @@ func getDefaultOLSConfigCR() *olsv1alpha1.OLSConfig { DefaultModel: "testModel", DefaultProvider: "testProvider", LogLevel: "INFO", + ConversationCache: olsv1alpha1.ConversationCacheSpec{ + Type: "postgres", + Postgres: olsv1alpha1.PostgresSpec{ + MaxConnections: 2000, + }, + }, }, }, } diff --git a/internal/controller/ols_app_server_deployment.go b/internal/controller/ols_app_server_deployment.go index d7b0d7d0..ba3d9a8b 100644 --- a/internal/controller/ols_app_server_deployment.go +++ b/internal/controller/ols_app_server_deployment.go @@ -76,11 +76,14 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( credentialMountPath := path.Join(APIKeyMountRoot, provider.CredentialsSecretRef.Name) secretMounts[provider.CredentialsSecretRef.Name] = credentialMountPath } - // TODO: Update DB - // Redis volume - //redisSecretName := cr.Spec.OLSConfig.ConversationCache.Redis.CredentialsSecret - //redisCredentialsMountPath := path.Join(CredentialsMountRoot, redisSecretName) - //secretMounts[redisSecretName] = redisCredentialsMountPath + + // Postgres Volume + postgresSecretName := PostgresSecretName + if cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret != "" { + postgresSecretName = cr.Spec.OLSConfig.ConversationCache.Postgres.CredentialsSecret + } + postgresCredentialsMountPath := path.Join(CredentialsMountRoot, postgresSecretName) + secretMounts[postgresSecretName] = postgresCredentialsMountPath // TLS volume if cr.Spec.OLSConfig.TLSConfig != nil && cr.Spec.OLSConfig.TLSConfig.KeyCertSecretRef.Name != "" { @@ -125,15 +128,16 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( }, } volumes = append(volumes, olsConfigVolume) - olsUserDataVolume := corev1.Volume{ - Name: OLSUserDataVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - } if dataCollectorEnabled { + olsUserDataVolume := corev1.Volume{ + Name: OLSUserDataVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } volumes = append(volumes, olsUserDataVolume) } +<<<<<<< HEAD // User provided additional CA certificates if cr.Spec.OLSConfig.AdditionalCAConfigMapRef != nil { @@ -157,6 +161,9 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( // TODO: Update DB //volumes = append(volumes, olsConfigVolume, olsUserDataVolume, getRedisCAConfigVolume()) +======= + volumes = append(volumes, getPostgresCAConfigVolume()) +>>>>>>> 7c4b826 (PostgreSQL integration) // mount the volumes of api keys secrets and OLS config map to the container volumeMounts := []corev1.VolumeMount{} @@ -195,9 +202,7 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( volumeMounts = append(volumeMounts, additionalCAVolumeMount, certBundleVolumeMount) } - // TODO: Update DB - //volumeMounts = append(volumeMounts, olsConfigVolumeMount, olsUserDataVolumeMount, getRedisCAVolumeMount(path.Join(OLSAppCertsMountRoot, RedisCertsSecretName, RedisCAVolume))) - + volumeMounts = append(volumeMounts, getPostgresCAVolumeMount(path.Join(OLSAppCertsMountRoot, PostgresCertsSecretName, PostgresCAVolume))) replicas := getOLSServerReplicas(cr) ols_server_resources := getOLSServerResources(cr) data_collector_resources := getOLSDataCollectorResources(cr) @@ -292,7 +297,7 @@ func (r *OLSConfigReconciler) generateOLSDeployment(cr *olsv1alpha1.OLSConfig) ( SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: &[]bool{false}[0], }, - VolumeMounts: []corev1.VolumeMount{olsUserDataVolumeMount, olsConfigVolumeMount}, + VolumeMounts: volumeMounts, Env: []corev1.EnvVar{ { Name: "OLS_CONFIG_FILE", @@ -316,6 +321,7 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD existingDeployment.Annotations[OLSConfigHashKey] != r.stateCache[OLSConfigHashStateCacheKey] || existingDeployment.Annotations[OLSAppTLSHashKey] != r.stateCache[OLSAppTLSHashStateCacheKey] || existingDeployment.Annotations[LLMProviderHashKey] != r.stateCache[LLMProviderHashStateCacheKey] || +<<<<<<< HEAD existingDeployment.Annotations[AdditionalCAHashKey] != r.stateCache[AdditionalCAHashStateCacheKey] { updateDeploymentAnnotations(existingDeployment, map[string]string{ OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], @@ -333,8 +339,22 @@ func (r *OLSConfigReconciler) updateOLSDeployment(ctx context.Context, existingD AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], // TODO: Update DB //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], +======= + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { + updateDeploymentAnnotations(existingDeployment, map[string]string{ + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], + }) + // update the deployment template annotation triggers the rolling update + updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], +>>>>>>> 7c4b826 (PostgreSQL integration) }) - changed = true } diff --git a/internal/controller/ols_app_server_reconciliator.go b/internal/controller/ols_app_server_reconciliator.go index 2afecb92..648d7a0c 100644 --- a/internal/controller/ols_app_server_reconciliator.go +++ b/internal/controller/ols_app_server_reconciliator.go @@ -92,8 +92,7 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols return fmt.Errorf("%s: %w", ErrCreateAPIConfigmap, err) } r.stateCache[OLSConfigHashStateCacheKey] = cm.Annotations[OLSConfigHashKey] - // TODO: Update DB - //r.stateCache[RedisConfigHashStateCacheKey] = cm.Annotations[RedisConfigHashKey] + r.stateCache[PostgresConfigHashStateCacheKey] = cm.Annotations[PostgresConfigHashKey] return nil @@ -107,8 +106,7 @@ func (r *OLSConfigReconciler) reconcileOLSConfigMap(ctx context.Context, cr *ols // update the state cache with the hash of the existing configmap. // so that we can skip the reconciling the deployment if the configmap has not changed. r.stateCache[OLSConfigHashStateCacheKey] = cm.Annotations[OLSConfigHashKey] - // TODO: Update DB - //r.stateCache[RedisConfigHashStateCacheKey] = cm.Annotations[RedisConfigHashKey] + r.stateCache[PostgresConfigHashStateCacheKey] = cm.Annotations[PostgresConfigHashKey] if foundCmHash == cm.Annotations[OLSConfigHashKey] { r.logger.Info("OLS configmap reconciliation skipped", "configmap", foundCm.Name, "hash", foundCm.Annotations[OLSConfigHashKey]) return nil @@ -247,20 +245,16 @@ func (r *OLSConfigReconciler) reconcileDeployment(ctx context.Context, cr *olsv1 err = r.Client.Get(ctx, client.ObjectKey{Name: OLSAppServerDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { updateDeploymentAnnotations(desiredDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], - AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], - // TODO: Update DB - //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], }) updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ - OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], - OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], - LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], - AdditionalCAHashKey: r.stateCache[AdditionalCAHashStateCacheKey], - // TODO: Update DB - //RedisSecretHashKey: r.stateCache[RedisSecretHashStateCacheKey], + OLSConfigHashKey: r.stateCache[OLSConfigHashStateCacheKey], + OLSAppTLSHashKey: r.stateCache[OLSAppTLSHashStateCacheKey], + LLMProviderHashKey: r.stateCache[LLMProviderHashStateCacheKey], + PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], }) r.logger.Info("creating a new deployment", "deployment", desiredDeployment.Name) err = r.Create(ctx, desiredDeployment) @@ -339,12 +333,12 @@ func (r *OLSConfigReconciler) reconcileLLMSecrets(ctx context.Context, cr *olsv1 annotateSecretWatcher(foundSecret) err = r.Update(ctx, foundSecret) if err != nil { - return fmt.Errorf("failed to update secret:%s. error: %w", foundSecret.Name, err) + return fmt.Errorf("%s: %s error: %w", ErrUpdateProviderSecret, foundSecret.Name, err) } } foundProviderCredentialsHash, err := hashBytes([]byte(providerCredentials)) if err != nil { - return fmt.Errorf("failed to generate OLS provider credentials hash %w", err) + return fmt.Errorf("%s: %w", ErrGenerateProviderCredentialsHash, err) } if foundProviderCredentialsHash == r.stateCache[LLMProviderHashStateCacheKey] { r.logger.Info("OLS llm secrets reconciliation skipped", "hash", foundProviderCredentialsHash) diff --git a/internal/controller/olsconfig_controller.go b/internal/controller/olsconfig_controller.go index 3d064eb4..7eb82a8e 100644 --- a/internal/controller/olsconfig_controller.go +++ b/internal/controller/olsconfig_controller.go @@ -59,11 +59,11 @@ type OLSConfigReconciler struct { } type OLSConfigReconcilerOptions struct { - LightspeedServiceImage string - LightspeedServiceRedisImage string - ConsoleUIImage string - Namespace string - ReconcileInterval time.Duration + LightspeedServiceImage string + LightspeedServicePostgresImage string + ConsoleUIImage string + Namespace string + ReconcileInterval time.Duration } // +kubebuilder:rbac:groups=ols.openshift.io,resources=olsconfigs,verbs=get;list;watch;create;update;patch;delete @@ -82,7 +82,7 @@ type OLSConfigReconcilerOptions struct { // OLM cannot create a role and rolebinding for a specific single namespace that is not the namespace the operator is installed in and/or watching // This has to be a cluster role // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch -// Secret access for redis server configuration +// Secret access for conversation cache server configuration // +kubebuilder:rbac:groups=core,namespace=openshift-lightspeed,resources=secrets,verbs=get;list;watch;create;update;patch;delete // Secret access for telemetry pull secret, must be a cluster role due to OLM limitations in managing roles in operator namespace // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch @@ -132,12 +132,7 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{RequeueAfter: 1 * time.Second}, err } r.logger.Info("reconciliation starts", "olsconfig generation", olsconfig.Generation) - // TODO: Update DB - // err = r.reconcileRedisServer(ctx, olsconfig) - // if err != nil { - // r.logger.Error(err, "Failed to reconcile ols redis") - // return ctrl.Result{}, err - // } + err = r.reconcileConsoleUI(ctx, olsconfig) if err != nil { r.logger.Error(err, "Failed to reconcile console UI") @@ -147,6 +142,15 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Update status condition for Console Plugin r.updateStatusCondition(ctx, olsconfig, typeConsolePluginReady, true, "All components are successfully deployed", nil) + err = r.reconcilePostgresServer(ctx, olsconfig) + if err != nil { + r.logger.Error(err, "Failed to reconcile ols postgres") + r.updateStatusCondition(ctx, olsconfig, typeCRReconciled, false, "Failed", nil) + return ctrl.Result{}, err + } + // Update status condition for Postgres cache + r.updateStatusCondition(ctx, olsconfig, typeCacheReady, true, "All components are successfully deployed", nil) + err = r.reconcileLLMSecrets(ctx, olsconfig) if err != nil { r.logger.Error(err, "Failed to reconcile LLM Provider Secrets") diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index e7a3a022..983cef36 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -140,10 +140,10 @@ var _ = BeforeSuite(func() { reconciler = &OLSConfigReconciler{ Options: OLSConfigReconcilerOptions{ - LightspeedServiceImage: "lightspeed-service-api:latest", - LightspeedServiceRedisImage: "lightspeed-service-redis:latest", - ConsoleUIImage: ConsoleUIImageDefault, - Namespace: OLSNamespaceDefault, + LightspeedServiceImage: OLSAppServerImageDefault, + LightspeedServicePostgresImage: PostgresServerImageDefault, + ConsoleUIImage: ConsoleUIImageDefault, + Namespace: OLSNamespaceDefault, }, logger: logf.Log.WithName("olsconfig.reconciler"), Client: k8sClient, diff --git a/internal/controller/types.go b/internal/controller/types.go index 026a44c9..eb2d2791 100644 --- a/internal/controller/types.go +++ b/internal/controller/types.go @@ -3,8 +3,6 @@ package controller import ( "context" - "k8s.io/apimachinery/pkg/util/intstr" - olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" ) @@ -109,13 +107,10 @@ type LoggingConfig struct { } type ConversationCacheConfig struct { - // Type of cache to use. Default: "redis" - Type string `json:"type" default:"redis"` - // TODO: Update DB - // Redis cache configuration - //Redis RedisCacheConfig `json:"redis,omitempty"` - // Memory cache configuration - Memory MemoryCacheConfig `json:"memory,omitempty"` + // Type of cache to use. Default: "postgres" + Type string `json:"type" default:"postgres"` + // Postgres cache configuration + Postgres PostgresCacheConfig `json:"postgres,omitempty"` } type MemoryCacheConfig struct { @@ -123,18 +118,20 @@ type MemoryCacheConfig struct { MaxEntries int `json:"max_entries,omitempty" default:"1000"` } -type RedisCacheConfig struct { - // Redis host - Host string `json:"host,omitempty" default:"lightspeed-redis-server.openshift-lightspeed.svc"` - // Redis port - Port int `json:"port,omitempty" default:"6379"` - // Redis maxmemory - MaxMemory *intstr.IntOrString `json:"max_memory,omitempty" default:"1024mb"` - // Redis maxmemory policy - MaxMemoryPolicy string `json:"max_memory_policy,omitempty" default:"allkeys-lru"` - // Path to the file containing redis credentials in the app server container. +type PostgresCacheConfig struct { + // Postgres host + Host string `json:"host,omitempty" default:"lightspeed-postgres-server.openshift-lightspeed.svc"` + // Postgres port + Port int `json:"port,omitempty" default:"5432"` + // Postgres user + User string `json:"user,omitempty" default:"postgres"` + // Postgres dbname + DbName string `json:"dbname,omitempty" default:"postgres"` + // Path to the file containing postgres credentials in the app server container PasswordPath string `json:"password_path,omitempty"` - // Redis CA certificate path + // SSLMode is the preferred ssl mode to connect with postgres + SSLMode string `json:"ssl_mode,omitempty" default:"require"` + // Postgres CA certificate path CACertPath string `json:"ca_cert_path,omitempty"` } diff --git a/internal/controller/utils.go b/internal/controller/utils.go index f5b81334..1f52729e 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -102,19 +102,19 @@ func setVolumeMounts(deployment *appsv1.Deployment, desiredVolumeMounts []corev1 return false, nil } -// TODO: Update DB -// setCommand sets the command for a specific container in a given deployment. -// func setCommand(deployment *appsv1.Deployment, desiredCommand []string, containerName string) (bool, error) { -// containerIndex, err := getContainerIndex(deployment, containerName) -// if err != nil { -// return false, err -// } -// if !apiequality.Semantic.DeepEqual(deployment.Spec.Template.Spec.Containers[containerIndex].Command, desiredCommand) { -// deployment.Spec.Template.Spec.Containers[containerIndex].Command = desiredCommand -// return true, nil -// } -// return false, nil -// } +// setDeploymentContainerEnvs sets the envs for a specific container in a given deployment. +func setDeploymentContainerEnvs(deployment *appsv1.Deployment, desiredEnvs []corev1.EnvVar, containerName string) (bool, error) { + containerIndex, err := getContainerIndex(deployment, containerName) + if err != nil { + return false, err + } + existingEnvs := deployment.Spec.Template.Spec.Containers[containerIndex].Env + if !apiequality.Semantic.DeepEqual(existingEnvs, desiredEnvs) { + deployment.Spec.Template.Spec.Containers[containerIndex].Env = desiredEnvs + return true, nil + } + return false, nil +} // setDeploymentContainerResources sets the resource requirements for a specific container in a given deployment. func setDeploymentContainerResources(deployment *appsv1.Deployment, resources *corev1.ResourceRequirements, containerName string) (bool, error) { diff --git a/test/e2e/assets.go b/test/e2e/assets.go index 23d6bb34..9176ef52 100644 --- a/test/e2e/assets.go +++ b/test/e2e/assets.go @@ -6,7 +6,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" olsv1alpha1 "github.com/openshift/lightspeed-operator/api/v1alpha1" ) @@ -41,7 +40,6 @@ func generateOLSConfig() (*olsv1alpha1.OLSConfig, error) { // nolint:unused llmType = LLMDefaultType } replicas := int32(1) - maxMemory := intstr.Parse("100mb") return &olsv1alpha1.OLSConfig{ ObjectMeta: metav1.ObjectMeta{ Name: OLSCRName, @@ -65,10 +63,10 @@ func generateOLSConfig() (*olsv1alpha1.OLSConfig, error) { // nolint:unused }, OLSConfig: olsv1alpha1.OLSSpec{ ConversationCache: olsv1alpha1.ConversationCacheSpec{ - Type: olsv1alpha1.Redis, - Redis: olsv1alpha1.RedisSpec{ - MaxMemory: &maxMemory, - MaxMemoryPolicy: "allkeys-lru", + Type: olsv1alpha1.Postgres, + Postgres: olsv1alpha1.PostgresSpec{ + SharedBuffers: "256MB", + MaxConnections: 2000, }, }, DefaultModel: llmModel, diff --git a/test/e2e/client.go b/test/e2e/client.go index 93e71be3..6ecf0f5d 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -36,7 +36,7 @@ const ( // DefaultPollInterval is the default interval for polling DefaultPollInterval = 5 * time.Second // DefaultPollTimeout is the default timeout for polling - DefaultPollTimeout = 10 * time.Minute + DefaultPollTimeout = 20 * time.Minute ) type ClientOptions struct {