Skip to content

Commit

Permalink
Add remote control plane support for SLI-exporter
Browse files Browse the repository at this point in the history 
This adds support for remote control planes for the SLI exporter. It now
takes an additional `KUBECONFIG` env var to connect to the control plane.

Additionally to determine the state of the maintenance of the service
cluster, it connects to the local service cluster to reconcile on
`UpgradeJob` objects.

Also the SLI-Prober doesn't need any configuration about the services
anymore. It will detect these on startup.
  • Loading branch information
@Kidswiss
Kidswiss committed Dec 6, 2024
1 parent aa73d55 commit 36762a3
Show file tree
Hide file tree
Showing 24 changed files with 323 additions and 137 deletions.
4 changes: 4 additions & 0 deletions apis/vshn/v1/dbaas_vshn_mariadb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,10 @@ type XVSHNMariaDB struct {
Status XVSHNMariaDBStatus `json:"status,omitempty"`
}

func (v *XVSHNMariaDB) GetInstanceNamespace() string {
return fmt.Sprintf("vshn-mariadb-%s", v.GetName())
}

// XVSHNMariaDBSpec defines the desired state of a VSHNMariaDB.
type XVSHNMariaDBSpec struct {
// Parameters are the configurable fields of a VSHNMariaDB.
Expand Down
5 changes: 5 additions & 0 deletions apis/vshn/v1/dbaas_vshn_redis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package v1

import (
"fmt"

xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand Down Expand Up @@ -166,6 +167,10 @@ type XVSHNRedis struct {
Status XVSHNRedisStatus `json:"status,omitempty"`
}

func (v *XVSHNRedis) GetInstanceNamespace() string {
return fmt.Sprintf("vshn-redis-%s", v.GetName())
}

// XVSHNRedisSpec defines the desired state of a VSHNRedis.
type XVSHNRedisSpec struct {
// Parameters are the configurable fields of a VSHNRedis.
Expand Down
5 changes: 5 additions & 0 deletions apis/vshn/v1/vshn_minio.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package v1

import (
"fmt"

xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand Down Expand Up @@ -130,6 +131,10 @@ type XVSHNMinio struct {
Status XVSHNMinioStatus `json:"status,omitempty"`
}

func (v *XVSHNMinio) GetInstanceNamespace() string {
return fmt.Sprintf("vshn-minio-%s", v.GetName())
}

// XVSHNMinioSpec defines the desired state of a VSHNMinio.
type XVSHNMinioSpec struct {
// Parameters are the configurable fields of a VSHNMinio.
Expand Down
120 changes: 90 additions & 30 deletions cmd/sliexporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@ package cmd

import (
"os"
"strconv"
"time"

managedupgradev1beta1 "github.com/appuio/openshift-upgrade-controller/api/v1beta1"
"github.com/go-logr/logr"
"github.com/spf13/cobra"
vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
"github.com/vshn/appcat/v4/pkg"
"github.com/vshn/appcat/v4/pkg/common/utils"
maintenancecontroller "github.com/vshn/appcat/v4/pkg/sliexporter/maintenance_controller"
"github.com/vshn/appcat/v4/pkg/sliexporter/probes"
vshnkeycloakcontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnkeycloak_controller"
Expand All @@ -16,16 +18,20 @@ import (
vshnpostgresqlcontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnpostgresql_controller"
vshnrediscontroller "github.com/vshn/appcat/v4/pkg/sliexporter/vshnredis_controller"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
)

type sliProber struct {
scheme *runtime.Scheme
metricsAddr, probeAddr string
leaderElect, enableVSHNPostgreSQL, enableVSHNRedis, enableVSHNMinio, enableMaintenanceStatus, enableKeycloak, enableMariaDB bool
scheme *runtime.Scheme
metricsAddr, probeAddr, serviceKubeConfig string
leaderElect bool
}

var s = sliProber{
Expand All @@ -39,21 +45,17 @@ var SLIProberCMD = &cobra.Command{
RunE: s.executeSLIProber,
}

const (
startupGraceMin = 10
)

func init() {
SLIProberCMD.Flags().StringVar(&s.metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
SLIProberCMD.Flags().StringVar(&s.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
SLIProberCMD.Flags().BoolVar(&s.leaderElect, "leader-elect", false, "Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
SLIProberCMD.Flags().BoolVar(&s.enableVSHNPostgreSQL, "vshn-postgresql", getEnvBool("APPCAT_SLI_VSHNPOSTGRESQL"),
"Enable probing of VSHNPostgreSQL instances")
SLIProberCMD.Flags().BoolVar(&s.enableVSHNRedis, "vshn-redis", getEnvBool("APPCAT_SLI_VSHNREDIS"),
"Enable probing of VSHNRedis instances")
SLIProberCMD.Flags().BoolVar(&s.enableVSHNMinio, "vshn-minio", getEnvBool("APPCAT_SLI_VSHNMINIO"),
"Enable probing of VSHNMinio instances")
SLIProberCMD.Flags().BoolVar(&s.enableMaintenanceStatus, "vshn-track-oc-maintenance-status", getEnvBool("APPCAT_SLI_TRACK_OC_MAINTENANCE_STATUS"),
"Enable oc maintenance status observer. Will set the labels 'maintenance' accordingly.")
SLIProberCMD.Flags().BoolVar(&s.enableKeycloak, "vshn-keycloak", getEnvBool("APPCAT_SLI_VSHNKEYCLOAK"), "Enable probing of VSHNKeycloak instances")
SLIProberCMD.Flags().BoolVar(&s.enableMariaDB, "vshn-mariadb", getEnvBool("APPCAT_SLI_VSHNMARIADB"), "Enable probing of VSHNMariaDB instances")
SLIProberCMD.Flags().StringVar(&s.serviceKubeConfig, "service-kubeconfig", os.Getenv("SERVICE_KUBECONFIG"),
"Kubeconfig for the service cluster itself, usually only for debugging purpose. The sliprober should run on the service clusters and thus use the in-cluster-config.")
}

func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
Expand All @@ -74,8 +76,20 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
return err
}

config, err := getServiceClusterConfig()
if err != nil {
return err
}

scClient, err := client.New(config, client.Options{
Scheme: pkg.SetupScheme(),
})
if err != nil {
return err
}

maintenanceRecociler := &maintenancecontroller.MaintenanceReconciler{
Client: mgr.GetClient(),
Client: scClient,
}

probeManager := probes.NewManager(log, maintenanceRecociler)
Expand All @@ -86,74 +100,88 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
return err
}

if s.enableVSHNPostgreSQL {
if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNPostgreSQL", ctrl.GetConfigOrDie()) {
log.Info("Enabling VSHNPostgreSQL controller")
if err = (&vshnpostgresqlcontroller.VSHNPostgreSQLReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ProbeManager: &probeManager,
StartupGracePeriod: 15 * time.Minute,
StartupGracePeriod: startupGraceMin * time.Minute,
PostgreDialer: probes.NewPostgreSQL,
ScClient: scClient,
}).SetupWithManager(mgr); err != nil {
log.Error(err, "unable to create controller", "controller", "VSHNPostgreSQL")
return err
}
}
if s.enableVSHNRedis {
if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNRedis", ctrl.GetConfigOrDie()) {
log.Info("Enabling VSHNRedis controller")
if err = (&vshnrediscontroller.VSHNRedisReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ProbeManager: &probeManager,
StartupGracePeriod: 10 * time.Minute,
StartupGracePeriod: startupGraceMin * time.Minute,
RedisDialer: probes.NewRedis,
ScClient: scClient,
}).SetupWithManager(mgr); err != nil {
log.Error(err, "unable to create controller", "controller", "VSHNRedis")
return err
}
}

if s.enableVSHNMinio {
if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMinio", ctrl.GetConfigOrDie()) {
log.Info("Enabling VSHNRedis controller")
if err = (&vshnminiocontroller.VSHNMinioReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ProbeManager: &probeManager,
StartupGracePeriod: 10 * time.Minute,
StartupGracePeriod: startupGraceMin * time.Minute,
MinioDialer: probes.NewMinio,
ScClient: scClient,
}).SetupWithManager(mgr); err != nil {
log.Error(err, "unable to create controller", "controller", "VSHNRedis")
return err
}
}

if s.enableKeycloak {
if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNKeycloak", ctrl.GetConfigOrDie()) {
log.Info("Enablign VSHNKeycloak controller")
if err = (&vshnkeycloakcontroller.VSHNKeycloakReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ProbeManager: &probeManager,
StartupGracePeriod: 10 * time.Minute,
StartupGracePeriod: startupGraceMin * time.Minute,
ScClient: scClient,
}).SetupWithManager(mgr); err != nil {
log.Error(err, "unable to create controller", "controller", "VSHNKeycloak")
return err
}
}

if s.enableMaintenanceStatus {
if utils.IsKindAvailable(managedupgradev1beta1.GroupVersion, "UpgradeJob", config) {
log.Info("Enable OC maintenance observer")
if err = maintenanceRecociler.SetupWithManager(mgr); err != nil {
serviceCluster, err := getServiceCluster(config)
if err != nil {
return err
}
if err = maintenanceRecociler.SetupWithManager(mgr, serviceCluster); err != nil {
log.Error(err, "unable to create controller", "controller", "Maintenance Observer")
return err
}
err = mgr.Add(*serviceCluster)
if err != nil {
return err
}
}
if s.enableMariaDB {
if utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMariaDB", ctrl.GetConfigOrDie()) {
log.Info("Enabling VSHNMariaDB controller")
if err = (&vshnmariadbcontroller.VSHNMariaDBReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ProbeManager: &probeManager,
StartupGracePeriod: 1 * time.Minute,
StartupGracePeriod: startupGraceMin * time.Minute,
MariaDBDialer: probes.NewMariaDB,
ScClient: scClient,
}).SetupWithManager(mgr); err != nil {
log.Error(err, "unable to create controller", "controller", "VSHNMariadb")
return err
Expand All @@ -179,7 +207,39 @@ func (s *sliProber) executeSLIProber(cmd *cobra.Command, _ []string) error {
return nil
}

func getEnvBool(key string) bool {
b, err := strconv.ParseBool(os.Getenv(key))
return err == nil && b
func getServiceCluster(config *rest.Config) (*cluster.Cluster, error) {
serviceCluster, err := cluster.New(config, func(o *cluster.Options) {
o.Scheme = pkg.SetupScheme()
})
return &serviceCluster, err
}

// getServiceClusterConfig will create an incluster config by default.
// If serviceKubeConfig is set, it will use that instead.
func getServiceClusterConfig() (*rest.Config, error) {

if s.serviceKubeConfig != "" {
kubeconfig, err := os.ReadFile(s.serviceKubeConfig)
if err != nil {
return nil, err
}

clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
if err != nil {
return nil, err
}

config, err := clientConfig.ClientConfig()
if err != nil {
return nil, err
}
return config, nil
}

config, err := rest.InClusterConfig()
if err != nil {
return nil, err
}

return config, nil
}
4 changes: 2 additions & 2 deletions pkg/apiserver/appcat/appcat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package appcat
import (
crossplane "github.com/crossplane/crossplane/apis/apiextensions/v1"
appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
"github.com/vshn/appcat/v4/pkg/apiserver"
"github.com/vshn/appcat/v4/pkg/apiserver/noop"
"github.com/vshn/appcat/v4/pkg/common/utils"
"k8s.io/apimachinery/pkg/runtime"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/registry/rest"
Expand Down Expand Up @@ -37,7 +37,7 @@ func New() restbuilder.ResourceHandlerProvider {

noopImplementation := noop.New(s, &appcatv1.AppCat{}, &appcatv1.AppCatList{})

if !apiserver.IsTypeAvailable(crossplane.SchemeGroupVersion.String(), "Composition") {
if !utils.IsKindAvailable(crossplane.SchemeGroupVersion, "Composition", loopback.GetLoopbackMasterClientConfig()) {
return noopImplementation, nil
}

Expand Down
20 changes: 0 additions & 20 deletions pkg/apiserver/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import (

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/discovery"
"sigs.k8s.io/apiserver-runtime/pkg/util/loopback"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand Down Expand Up @@ -91,21 +89,3 @@ func GetBackupColumnDefinition() []metav1.TableColumnDefinition {
{Name: "Age", Type: "date", Description: desc["creationTimestamp"]},
}
}

func IsTypeAvailable(gv string, kind string) bool {
d, err := discovery.NewDiscoveryClientForConfig(loopback.GetLoopbackMasterClientConfig())
if err != nil {
return false
}
resources, err := d.ServerResourcesForGroupVersion(gv)
if err != nil {
return false
}

for _, res := range resources.APIResources {
if res.Kind == kind {
return true
}
}
return false
}
4 changes: 2 additions & 2 deletions pkg/apiserver/vshn/mariadb/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import (
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
"github.com/vshn/appcat/v4/pkg/apiserver"
"github.com/vshn/appcat/v4/pkg/apiserver/noop"
"github.com/vshn/appcat/v4/pkg/apiserver/vshn/k8up"
"github.com/vshn/appcat/v4/pkg/common/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
Expand Down Expand Up @@ -37,7 +37,7 @@ func New() restbuilder.ResourceHandlerProvider {

noopImplementation := noop.New(s, &appcatv1.VSHNMariaDBBackup{}, &appcatv1.VSHNMariaDBBackupList{})

if !apiserver.IsTypeAvailable(vshnv1.GroupVersion.String(), "XVSHNMariaDB") {
if !utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNMariaDB", loopback.GetLoopbackMasterClientConfig()) {
return noopImplementation, nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/apiserver/vshn/nextcloud/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
k8upv1 "github.com/k8up-io/k8up/v2/api/v1"
appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
"github.com/vshn/appcat/v4/pkg/apiserver"
"github.com/vshn/appcat/v4/pkg/apiserver/noop"
"github.com/vshn/appcat/v4/pkg/apiserver/vshn/k8up"
"github.com/vshn/appcat/v4/pkg/apiserver/vshn/postgres"
"github.com/vshn/appcat/v4/pkg/common/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
Expand Down Expand Up @@ -41,7 +41,7 @@ func New() restbuilder.ResourceHandlerProvider {

noopImplementation := noop.New(s, &appcatv1.VSHNNextcloudBackup{}, &appcatv1.VSHNNextcloudBackupList{})

if !apiserver.IsTypeAvailable(vshnv1.GroupVersion.String(), "XVSHNNextcloud") {
if !utils.IsKindAvailable(vshnv1.GroupVersion, "XVSHNNextcloud", loopback.GetLoopbackMasterClientConfig()) {
return noopImplementation, nil
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/apiserver/vshn/postgres/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package postgres
import (
appcatv1 "github.com/vshn/appcat/v4/apis/apiserver/v1"
vshnv1 "github.com/vshn/appcat/v4/apis/vshn/v1"
"github.com/vshn/appcat/v4/pkg/apiserver"
"github.com/vshn/appcat/v4/pkg/apiserver/noop"
"github.com/vshn/appcat/v4/pkg/common/utils"
"k8s.io/apimachinery/pkg/runtime"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/registry/rest"
Expand All @@ -29,7 +29,7 @@ func New() restbuilder.ResourceHandlerProvider {

noopImplementation := noop.New(s, &appcatv1.VSHNPostgresBackup{}, &appcatv1.VSHNPostgresBackupList{})

if !apiserver.IsTypeAvailable(vshnv1.GroupVersion.String(), "VSHNPostgreSQL") {
if !utils.IsKindAvailable(vshnv1.GroupVersion, "VSHNPostgreSQL", loopback.GetLoopbackMasterClientConfig()) {
return noopImplementation, nil
}

Expand Down
Loading

0 comments on commit 36762a3

Please sign in to comment.