diff --git a/tests/integration/helm_default_installation_test.go b/tests/integration/helm_default_installation_test.go index 721a890b56..e7a7d4fe5c 100644 --- a/tests/integration/helm_default_installation_test.go +++ b/tests/integration/helm_default_installation_test.go @@ -34,9 +34,7 @@ func Test_Helm_Default_FluentD_Metadata(t *testing.T) { waitDuration = 3 * time.Minute logsGeneratorCount uint = 1000 ) - var ( - expectedMetrics = internal.DefaultExpectedMetrics - ) + expectedMetrics := internal.DefaultExpectedMetrics // TODO: // Refactor this: we should find a way to inject this into step func helpers diff --git a/tests/integration/helm_otc_metadata_installation_test.go b/tests/integration/helm_otc_metadata_installation_test.go index fe38d5bfb2..efb5ce99ac 100644 --- a/tests/integration/helm_otc_metadata_installation_test.go +++ b/tests/integration/helm_otc_metadata_installation_test.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" log "k8s.io/klog/v2" + "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" @@ -25,7 +26,6 @@ import ( "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/ctxopts" "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/receivermock" "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/stepfuncs" - "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/strings" ) func Test_Helm_Default_OT_Metadata(t *testing.T) { @@ -35,15 +35,7 @@ func Test_Helm_Default_OT_Metadata(t *testing.T) { logsGeneratorCount uint = 1000 ) - var ( - expectedMetrics = internal.DefaultExpectedMetrics - ) - - // TODO: - // Refactor this: we should find a way to inject this into step func helpers - // like stepfuncs.WaitUntilPodsAvailable() instead of relying on an implementation - // detail. - releaseName := strings.ReleaseNameFromT(t) + expectedMetrics := internal.DefaultExpectedMetrics featInstall := features.New("installation"). Assess("sumologic secret is created", @@ -53,80 +45,103 @@ func Test_Helm_Default_OT_Metadata(t *testing.T) { require.Len(t, secret.Data, 10) return ctx }). - Assess("otelcol logs pods are available", - stepfuncs.WaitUntilPodsAvailable( - v1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s-sumologic-otelcol-logs", releaseName), - }, - 3, + Assess("otelcol logs statefulset is ready", + stepfuncs.WaitUntilStatefulSetIsReady( waitDuration, tickDuration, + stepfuncs.WithNameF( + stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-logs"), + ), + stepfuncs.WithLabelsF( + stepfuncs.LabelFormatterKV{ + K: "app", + V: stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-logs"), + }, + ), ), ). - Assess("otelcol logs buffers PVCs are created", + Assess("otelcol logs buffers PVCs are created and bound", func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context { - namespace := ctxopts.Namespace(ctx) - releaseName := ctxopts.HelmRelease(ctx) - kubectlOptions := ctxopts.KubectlOptions(ctx) - - t.Logf("kubeconfig: %s", kubectlOptions.ConfigPath) - cl, err := terrak8s.GetKubernetesClientFromOptionsE(t, kubectlOptions) - require.NoError(t, err) - - assert.Eventually(t, func() bool { - pvcs, err := cl.CoreV1().PersistentVolumeClaims(namespace). - List(ctx, v1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s-sumologic-otelcol-logs", releaseName), - }) - if !assert.NoError(t, err) { - return false - } - - return err == nil && len(pvcs.Items) == 3 - }, waitDuration, tickDuration) + res := envConf.Client().Resources(ctxopts.Namespace(ctx)) + pvcs := corev1.PersistentVolumeClaimList{} + cond := conditions. + New(res). + ResourceListMatchN(&pvcs, 3, + func(object k8s.Object) bool { + pvc := object.(*corev1.PersistentVolumeClaim) + if pvc.Status.Phase != corev1.ClaimBound { + log.V(0).Infof("PVC %q not bound yet", pvc.Name) + return false + } + return true + }, + resources.WithLabelSelector( + fmt.Sprintf("app=%s-sumologic-otelcol-logs", ctxopts.HelmRelease(ctx)), + ), + ) + require.NoError(t, + wait.For(cond, + wait.WithTimeout(waitDuration), + wait.WithInterval(tickDuration), + ), + ) return ctx }). - Assess("otelcol metrics pods are available", - stepfuncs.WaitUntilPodsAvailable( - v1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s-sumologic-otelcol-metrics", releaseName), - }, - 3, + Assess("otelcol metrics statefulset is ready", + stepfuncs.WaitUntilStatefulSetIsReady( waitDuration, tickDuration, + stepfuncs.WithNameF( + stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-metrics"), + ), + stepfuncs.WithLabelsF( + stepfuncs.LabelFormatterKV{ + K: "app", + V: stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-metrics"), + }, + ), ), ). - Assess("otelcol metrics buffers PVCs are created", + Assess("otelcol metrics buffers PVCs are created and bound", func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context { - namespace := ctxopts.Namespace(ctx) - releaseName := ctxopts.HelmRelease(ctx) - kubectlOptions := ctxopts.KubectlOptions(ctx) - - t.Logf("kubeconfig: %s", kubectlOptions.ConfigPath) - cl, err := terrak8s.GetKubernetesClientFromOptionsE(t, kubectlOptions) - require.NoError(t, err) - - assert.Eventually(t, func() bool { - pvcs, err := cl.CoreV1().PersistentVolumeClaims(namespace). - List(ctx, v1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s-sumologic-otelcol-metrics", releaseName), - }) - if !assert.NoError(t, err) { - return false - } - - return len(pvcs.Items) == 3 - }, waitDuration, tickDuration) + res := envConf.Client().Resources(ctxopts.Namespace(ctx)) + pvcs := corev1.PersistentVolumeClaimList{} + cond := conditions. + New(res). + ResourceListMatchN(&pvcs, 3, + func(object k8s.Object) bool { + pvc := object.(*corev1.PersistentVolumeClaim) + if pvc.Status.Phase != corev1.ClaimBound { + log.V(0).Infof("PVC %q not bound yet", pvc.Name) + return false + } + return true + }, + resources.WithLabelSelector( + fmt.Sprintf("app=%s-sumologic-otelcol-metrics", ctxopts.HelmRelease(ctx)), + ), + ) + require.NoError(t, + wait.For(cond, + wait.WithTimeout(waitDuration), + wait.WithInterval(tickDuration), + ), + ) return ctx }). - Assess("fluentd events pod is available", - stepfuncs.WaitUntilPodsAvailable( - v1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s-sumologic-fluentd-events", releaseName), - }, - 1, + Assess("fluentd events statefulset is ready", + stepfuncs.WaitUntilStatefulSetIsReady( waitDuration, tickDuration, + stepfuncs.WithNameF( + stepfuncs.ReleaseFormatter("%s-sumologic-fluentd-events"), + ), + stepfuncs.WithLabelsF( + stepfuncs.LabelFormatterKV{ + K: "app", + V: stepfuncs.ReleaseFormatter("%s-sumologic-fluentd-events"), + }, + ), ), ). Assess("fluentd events buffers PVCs are created", diff --git a/tests/integration/internal/stepfuncs/assess_funcs.go b/tests/integration/internal/stepfuncs/assess_funcs.go index 8b0f38e638..4cb5065e04 100644 --- a/tests/integration/internal/stepfuncs/assess_funcs.go +++ b/tests/integration/internal/stepfuncs/assess_funcs.go @@ -6,14 +6,22 @@ import ( "testing" "time" - terrak8s "github.com/gruntwork-io/terratest/modules/k8s" - "github.com/gruntwork-io/terratest/modules/retry" - "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" log "k8s.io/klog/v2" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" + terrak8s "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/ctxopts" k8s_internal "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/k8s" "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/receivermock" @@ -123,3 +131,55 @@ func WaitUntilExpectedLogsPresent( return ctx } } + +// WaitUntilStatefulSetIsReady waits for a specified duration and check with the +// specified tick interval whether the stateful set (as described by the provided options) +// is ready. +// +// Readiness for a stateful set in here is defined as having N ready replicas where +// N is also equal to the spec replicas set on the stateful set. +func WaitUntilStatefulSetIsReady( + waitDuration time.Duration, + tickDuration time.Duration, + opts ...Option, +) features.Func { + return func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context { + sts := appsv1.StatefulSet{ + ObjectMeta: v1.ObjectMeta{ + Namespace: ctxopts.Namespace(ctx), + }, + } + + listOpts := []resources.ListOption{} + for _, opt := range opts { + opt.Apply(ctx, &sts) + listOpts = append(listOpts, opt.GetListOption(ctx)) + } + + res := envConf.Client().Resources(ctxopts.Namespace(ctx)) + cond := conditions. + New(res). + ResourceListMatchN(&appsv1.StatefulSetList{Items: []appsv1.StatefulSet{sts}}, + 1, + func(obj k8s.Object) bool { + sts := obj.(*appsv1.StatefulSet) + log.V(5).InfoS("StatefulSet", "status", sts.Status) + if *sts.Spec.Replicas != sts.Status.ReadyReplicas { + log.V(0).Infof("StatefulSet %q not yet fully ready", sts.Name) + return false + } + return true + }, + listOpts..., + ) + + require.NoError(t, + wait.For(cond, + wait.WithTimeout(waitDuration), + wait.WithInterval(tickDuration), + ), + ) + + return ctx + } +} diff --git a/tests/integration/internal/stepfuncs/options.go b/tests/integration/internal/stepfuncs/options.go new file mode 100644 index 0000000000..08588806ce --- /dev/null +++ b/tests/integration/internal/stepfuncs/options.go @@ -0,0 +1,138 @@ +package stepfuncs + +import ( + "context" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + + "github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/ctxopts" +) + +// Option is an interface that is used to pass in types that fulfill it to e.g. +// assess functions in internal/stepfuncs/assess_funcs.go so that their custom +// modification logic can be applied on k8s.Object. +// +// Example: +// +// func WaitUntilStatefulSetIsReady( +// opts ...Option, +// ) features.Func { +// return func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context { +// sts := appsv1.StatefulSet{ +// ObjectMeta: v1.ObjectMeta{ +// Namespace: ctxopts.Namespace(ctx), +// }, +// } +// for _, opt := range opts { +// opt.Apply(ctx, &sts) +// ... +// } +// ... +// } +// +type Option interface { + Apply(ctx context.Context, obj k8s.Object) + GetListOption(ctx context.Context) resources.ListOption +} + +// nameOption is an Option that sets a concrete name on the k8s.Object. +type nameOption struct { + name string +} + +func (no nameOption) Apply(ctx context.Context, obj k8s.Object) { + obj.SetName(no.name) +} + +func (no nameOption) GetListOption(ctx context.Context) resources.ListOption { + return func(lo *metav1.ListOptions) {} +} + +// WithName creates a nameOption with provided name. +func WithName(name string) Option { + return nameOption{ + name: name, + } +} + +// nameOption is an Option that allows setting k8s.Object's name using a formatter +// in order to e.g. include value from context that's passed into a running test. +type nameFOption struct { + formatter Formatter +} + +func (no nameFOption) Apply(ctx context.Context, obj k8s.Object) { + obj.SetName(no.formatter(ctx)) +} + +func (no nameFOption) GetListOption(ctx context.Context) resources.ListOption { + return func(lo *metav1.ListOptions) { + } +} + +// WithNameF creates a nameFOption using the provided formatter. +func WithNameF(formatter Formatter) Option { + return nameFOption{ + formatter: formatter, + } +} + +type LabelFormatterKV struct { + K string + V Formatter +} + +type labelsFOption struct { + kvs []LabelFormatterKV +} + +func (lo labelsFOption) Apply(ctx context.Context, obj k8s.Object) { + labels := make(map[string]string, len(lo.kvs)) + for _, elem := range lo.kvs { + labels[elem.K] = elem.V(ctx) + } + obj.SetLabels(labels) +} + +func (lo labelsFOption) GetListOption(ctx context.Context) resources.ListOption { + labels := make([]string, 0, len(lo.kvs)) + for _, elem := range lo.kvs { + labels = append(labels, fmt.Sprintf("%s=%s", elem.K, elem.V(ctx))) + } + + return resources.WithLabelSelector(strings.Join(labels, ",")) +} + +// WithLabelsF creates an Option which can be used to set key value pairs with +// custom value formatting. +// +// Example: +// stepfuncs.WaitUntilStatefulSetIsReady( +// ... +// stepfuncs.WithLabelsF( +// stepfuncs.LabelFormatterKV{ +// K: "app", +// V: stepfuncs.ReleaseFormatter("%s-sumologic-fluentd-events"), +// }, +// ), +// ), +// +// The above snippet will use the helm release name (passed around in tests context) +// and place it in the `%s` format string when a test will be executed. +func WithLabelsF(kvs ...LabelFormatterKV) Option { + return labelsFOption{ + kvs: kvs, + } +} + +type Formatter func(ctx context.Context) string + +func ReleaseFormatter(format string) Formatter { + return func(ctx context.Context) string { + return fmt.Sprintf(format, ctxopts.HelmRelease(ctx)) + } +}