Skip to content

Commit

Permalink
Add metrics correctness support to make + ci (#2289)
Browse files Browse the repository at this point in the history
Co-authored-by: Bogdan Drutu <bogdandrutu@gmail.com>
  • Loading branch information
pmcollins and bogdandrutu authored Jan 8, 2021
1 parent 953bf83 commit 6e6a088
Show file tree
Hide file tree
Showing 5 changed files with 165 additions and 7 deletions.
19 changes: 19 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,12 @@ workflows:
filters:
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
- correctness-metrics:
requires:
- cross-compile
filters:
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
- test:
requires:
- setup-environment
Expand All @@ -170,6 +176,7 @@ workflows:
requires:
- cross-compile
- correctness
- correctness-metrics
- lint
- loadtest
- test
Expand All @@ -182,6 +189,7 @@ workflows:
requires:
- cross-compile
- correctness
- correctness-metrics
- lint
- loadtest
- test
Expand All @@ -199,6 +207,7 @@ workflows:
requires:
- cross-compile
- correctness
- correctness-metrics
- lint
- loadtest
- test
Expand Down Expand Up @@ -305,6 +314,16 @@ jobs:
command: TEST_ARGS="-test.run=$(make -s testbed-list-correctness | circleci tests split|xargs echo|sed 's/ /|/g')" make testbed-correctness
- github_issue_generator

correctness-metrics:
executor: golang
parallelism: 4
resource_class: medium+
steps:
- attach_to_workspace
- run:
name: Metrics correctness tests
command: TEST_ARGS="-test.run=$(make -s testbed-list-correctness-metrics | circleci tests split|xargs echo|sed 's/ /|/g')" make testbed-correctness-metrics

test:
executor: golang
environment:
Expand Down
8 changes: 8 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,14 @@ testbed-list-loadtest:
testbed-list-correctness:
RUN_TESTBED=1 $(GOTEST) -v ./testbed/correctness --test.list '.*'| grep "^Test"

.PHONY: testbed-list-correctness-metrics
testbed-list-correctness-metrics:
RUN_TESTBED=1 $(GOTEST) -v ./testbed/correctness/metrics --test.list '.*'| grep "^TestHarness_"

.PHONY: testbed-correctness-metrics
testbed-correctness-metrics: otelcol
cd ./testbed/correctness/metrics && ./runtests.sh

.PHONY: gotest
gotest:
@$(MAKE) for-all CMD="make test"
Expand Down
37 changes: 30 additions & 7 deletions testbed/correctness/metrics/metrics_correctness_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,20 +27,36 @@ import (
"go.opentelemetry.io/collector/testbed/testbed"
)

func TestMetricsGoldenData(t *testing.T) {
tests, err := correctness.LoadPictOutputPipelineDefs("../testdata/generated_pict_pairs_metrics_pipeline.txt")
// tests with the prefix "TestHarness_" get run in the "correctness-metrics" ci job
func TestHarness_MetricsGoldenData(t *testing.T) {
tests, err := correctness.LoadPictOutputPipelineDefs(
"../testdata/generated_pict_pairs_metrics_pipeline.txt",
)
require.NoError(t, err)

res := results{}
res.Init("results")
for _, test := range tests {
test.TestName = fmt.Sprintf("%s-%s", test.Receiver, test.Exporter)
test.DataSender = correctness.ConstructMetricsSender(t, test.Receiver)
test.DataReceiver = correctness.ConstructReceiver(t, test.Exporter)
t.Run(test.TestName, func(t *testing.T) {
testWithMetricsGoldenDataset(t, test.DataSender.(testbed.MetricDataSender), test.DataReceiver)
r := testWithMetricsGoldenDataset(
t,
test.DataSender.(testbed.MetricDataSender),
test.DataReceiver,
)
res.Add("", r)
})
}
res.Save()
}

func testWithMetricsGoldenDataset(t *testing.T, sender testbed.MetricDataSender, receiver testbed.DataReceiver) {
func testWithMetricsGoldenDataset(
t *testing.T,
sender testbed.MetricDataSender,
receiver testbed.DataReceiver,
) result {
mds := getTestMetrics(t)
accumulator := newDiffAccumulator()
h := newTestHarness(
Expand All @@ -62,9 +78,16 @@ func testWithMetricsGoldenDataset(t *testing.T, sender testbed.MetricDataSender,
tc.stopTestbedReceiver()
tc.stopCollector()

if accumulator.foundDiffs {
r := result{
testName: t.Name(),
testResult: "PASS",
numDiffs: accumulator.numDiffs,
}
if accumulator.numDiffs > 0 {
r.testResult = "FAIL"
t.Fail()
}
return r
}

func getTestMetrics(t *testing.T) []pdata.Metrics {
Expand All @@ -75,7 +98,7 @@ func getTestMetrics(t *testing.T) []pdata.Metrics {
}

type diffAccumulator struct {
foundDiffs bool
numDiffs int
}

var _ diffConsumer = (*diffAccumulator)(nil)
Expand All @@ -86,7 +109,7 @@ func newDiffAccumulator() *diffAccumulator {

func (d *diffAccumulator) accept(metricName string, diffs []*MetricDiff) {
if len(diffs) > 0 {
d.foundDiffs = true
d.numDiffs++
log.Printf("Found diffs for [%v]\n%v", metricName, diffs)
}
}
73 changes: 73 additions & 0 deletions testbed/correctness/metrics/results.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package metrics

import (
"fmt"
"io"
"log"
"os"
"path"
"time"
)

type results struct {
resultsFile *os.File
}

type result struct {
testName string
testResult string
numDiffs int
}

func (r *results) Init(resultsDir string) {
err := os.MkdirAll(resultsDir, os.FileMode(0755))
if err != nil {
log.Fatalf(err.Error())
}
r.resultsFile, err = os.Create(path.Join(resultsDir, "CORRECTNESSRESULTS.md"))
if err != nil {
log.Fatalf(err.Error())
}
r.writeString(
"# Test Results\n" +
fmt.Sprintf("Started: %s\n\n", time.Now().Format(time.RFC1123Z)) +
"Test |Result|Num Diffs|\n" +
"----------------------------------------|------|--------:|\n",
)
}

func (r *results) Add(_ string, rslt interface{}) {
tr := rslt.(result)
line := fmt.Sprintf(
"%-40s|%-6s|%9d|\n",
tr.testName,
tr.testResult,
tr.numDiffs,
)
r.writeString(line)
}

func (r *results) writeString(s string) {
_, _ = io.WriteString(r.resultsFile, s)
}

func (r *results) Save() {
err := r.resultsFile.Close()
if err != nil {
panic(err)
}
}
35 changes: 35 additions & 0 deletions testbed/correctness/metrics/runtests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/bin/bash

# Copyright The OpenTelemetry Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

SED="sed"

PASS_COLOR=$(printf "\033[32mPASS\033[0m")
FAIL_COLOR=$(printf "\033[31mFAIL\033[0m")
TEST_COLORIZE="${SED} 's/PASS/${PASS_COLOR}/' | ${SED} 's/FAIL/${FAIL_COLOR}/'"
echo ${TEST_ARGS}
mkdir -p results
go test -v ${TEST_ARGS} 2>&1 | tee results/testoutput.log | bash -c "${TEST_COLORIZE}"

testStatus=${PIPESTATUS[0]}

mkdir -p results/junit
go-junit-report < results/testoutput.log > results/junit/results.xml

bash -c "cat results/CORRECTNESSRESULTS.md | ${TEST_COLORIZE}"

exit ${testStatus}

0 comments on commit 6e6a088

Please sign in to comment.