Skip to content

Commit

Permalink
refactor(tests): refactor integration tests so that assess fgiuncs ca…
Browse files Browse the repository at this point in the history
…n accept stepfuncs.Option (#2012)
  • Loading branch information
Mikołaj Świątek committed Jan 10, 2022
1 parent a8e9051 commit 9079118
Show file tree
Hide file tree
Showing 3 changed files with 234 additions and 2 deletions.
30 changes: 28 additions & 2 deletions deploy/helm/sumologic/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4186,7 +4186,33 @@ otellogs:
include_file_path: true
include_file_name: false
operators:

# Find out which format is used by kubernetes
- type: router
id: get-format
routes:
- output: parser-docker
expr: '$$body matches "^\\{"'
- output: parser-crio
expr: '$$body matches "^[^ Z]+ "'
- output: parser-containerd
expr: '$$body matches "^[^ Z]+Z"'
# Parse CRI-O format
- type: regex_parser
id: parser-crio
regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
output: extract-metadata-from-filepath
timestamp:
parse_from: time
layout_type: gotime
layout: '2006-01-02T15:04:05.000000000-07:00'
# Parse CRI-Containerd format
- type: regex_parser
id: parser-containerd
regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) (?P<log>.*)$'
output: extract-metadata-from-filepath
timestamp:
parse_from: time
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
## parser-docker interprets the input string as JSON and moves the `time` field from the JSON to Timestamp field in the OTLP log
## record.
# Input Body (string): '{"log":"2001-02-03 04:05:06 first line\n","stream":"stdout","time":"2021-11-25T09:59:13.23887954Z"}'
Expand All @@ -4195,7 +4221,7 @@ otellogs:
# Output Timestamp: 2021-11-25 09:59:13.23887954 +0000 UTC
- id: parser-docker
type: json_parser
# output: join-multipart-entries
output: extract-metadata-from-filepath
timestamp:
parse_from: time
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
Expand Down
152 changes: 152 additions & 0 deletions tests/integration/helm_otelcol_logs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package integration

import (
"context"
"fmt"
"testing"
"time"

"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/pkg/features"

terrak8s "github.com/gruntwork-io/terratest/modules/k8s"
"github.com/stretchr/testify/require"

corev1 "k8s.io/api/core/v1"
log "k8s.io/klog/v2"
"sigs.k8s.io/e2e-framework/klient/k8s"
"sigs.k8s.io/e2e-framework/klient/k8s/resources"
"sigs.k8s.io/e2e-framework/klient/wait"
"sigs.k8s.io/e2e-framework/klient/wait/conditions"

"github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal"
"github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/ctxopts"
"github.com/SumoLogic/sumologic-kubernetes-collection/tests/integration/internal/stepfuncs"
)

func Test_Helm_Otelcol_Logs(t *testing.T) {
const (
tickDuration = 3 * time.Second
waitDuration = 3 * time.Minute
logsGeneratorCount uint = 1000
)

featInstall := features.New("installation").
Assess("sumologic secret is created",
func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
terrak8s.WaitUntilSecretAvailable(t, ctxopts.KubectlOptions(ctx), "sumologic", 60, tickDuration)
secret := terrak8s.GetSecret(t, ctxopts.KubectlOptions(ctx), "sumologic")
require.Len(t, secret.Data, 2)
return ctx
}).
Assess("otelcol logs statefulset is ready",
stepfuncs.WaitUntilStatefulSetIsReady(
waitDuration,
tickDuration,
stepfuncs.WithNameF(
stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-logs"),
),
stepfuncs.WithLabelsF(
stepfuncs.LabelFormatterKV{
K: "app",
V: stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-logs"),
},
),
),
).
Assess("otelcol logs buffers PVCs are created and bound",
func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
res := envConf.Client().Resources(ctxopts.Namespace(ctx))
pvcs := corev1.PersistentVolumeClaimList{}
cond := conditions.
New(res).
ResourceListMatchN(&pvcs, 3,
func(object k8s.Object) bool {
pvc := object.(*corev1.PersistentVolumeClaim)
if pvc.Status.Phase != corev1.ClaimBound {
log.V(0).Infof("PVC %q not bound yet", pvc.Name)
return false
}
return true
},
resources.WithLabelSelector(
fmt.Sprintf("app=%s-sumologic-otelcol-logs", ctxopts.HelmRelease(ctx)),
),
)
require.NoError(t,
wait.For(cond,
wait.WithTimeout(waitDuration),
wait.WithInterval(tickDuration),
),
)
return ctx
}).
Assess("otelcol daemonset is ready",
stepfuncs.WaitUntilDaemonSetIsReady(
waitDuration,
tickDuration,
stepfuncs.WithNameF(
stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-logs-collector"),
),
stepfuncs.WithLabelsF(
stepfuncs.LabelFormatterKV{
K: "app",
V: stepfuncs.ReleaseFormatter("%s-sumologic-otelcol-logs-collector"),
},
),
),
).
Feature()
featLogs := features.New("logs").
Setup(stepfuncs.GenerateLogsWithDeployment(
logsGeneratorCount,
internal.LogsGeneratorName,
internal.LogsGeneratorNamespace,
internal.LogsGeneratorImage,
)).
Assess("logs from log generator present", stepfuncs.WaitUntilExpectedLogsPresent(
logsGeneratorCount,
map[string]string{
"namespace": internal.LogsGeneratorName,
"pod_labels_app": internal.LogsGeneratorName,
},
internal.ReceiverMockNamespace,
internal.ReceiverMockServiceName,
internal.ReceiverMockServicePort,
waitDuration,
tickDuration,
)).
Assess("expected container log metadata is present", stepfuncs.WaitUntilExpectedLogsPresent(
logsGeneratorCount,
map[string]string{
"_collector": "kubernetes",
"namespace": internal.LogsGeneratorName,
"pod_labels_app": internal.LogsGeneratorName,
"container": internal.LogsGeneratorName,
"deployment": internal.LogsGeneratorName,
"replicaset": "",
"pod": "",
"k8s.pod.id": "",
"k8s.pod.pod_name": "",
"k8s.container.id": "", // TODO: reconsider this, it's not reliable
"host": "",
"node": "",
},
internal.ReceiverMockNamespace,
internal.ReceiverMockServiceName,
internal.ReceiverMockServicePort,
waitDuration,
tickDuration,
)).
Teardown(
func(ctx context.Context, t *testing.T, envConf *envconf.Config) context.Context {
opts := *ctxopts.KubectlOptions(ctx)
opts.Namespace = internal.LogsGeneratorNamespace
terrak8s.RunKubectl(t, &opts, "delete", "deployment", internal.LogsGeneratorName)
return ctx
}).
Teardown(stepfuncs.KubectlDeleteNamespaceOpt(internal.LogsGeneratorNamespace)).
Feature()

testenv.Test(t, featInstall, featLogs)
}
54 changes: 54 additions & 0 deletions tests/integration/values/values_helm_otelcol_logs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
sumologic:
setupEnabled: true
accessId: "dummy"
accessKey: "dummy"
endpoint: http://receiver-mock.receiver-mock:3000/terraform/api/

logs:
metadata:
provider: otelcol

metrics:
enabled: false

# We're using otelcol instead
fluent-bit:
enabled: false

fluentd:
events:
enabled: false

# Request less resources so that this fits on Github actions runners environment
metadata:
persistence:
size: 128Mi
logs:
logLevel: debug
statefulset:
resources:
requests:
cpu: 100m
memory: 128Mi

otellogs:
enabled: true
config:
service:
pipelines:
logs/containers:
receivers:
- filelog/containers
exporters:
- otlphttp
processors:
- filter/exclude_receiver_mock_container
processors:
# Filter out receiver-mock logs to prevent snowball effect
filter/exclude_receiver_mock_container:
logs:
exclude:
match_type: strict
record_attributes:
- key: k8s.container.name
value: receiver-mock

0 comments on commit 9079118

Please sign in to comment.