Strip time, hostname, service columns out before parsing

The data we get from rsyslog includes more than just the raw JSON. Strip those bits out before trying to parse anything.
prometheus-community · Aug 30, 2017 · 7279dca · 7279dca
1 parent 040010c
commit 7279dca
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 54 deletions.
diff --git a/exporter.go b/exporter.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"log"
 	"os"
 	"sync"
@@ -37,7 +38,8 @@ func newRsyslogExporter() *rsyslogExporter {
 	return e
 }
 
-func (re *rsyslogExporter) handleStatLine(buf []byte) {
+func (re *rsyslogExporter) handleStatLine(rawbuf []byte) {
+	buf := bytes.SplitN(rawbuf, []byte(" "), 4)[3]
 	pstatType := getStatType(buf)
 	log.Printf("pstatType: %+v", pstatType)
 

diff --git a/exporter_test.go b/exporter_test.go
@@ -50,8 +50,8 @@ func testHelper(t *testing.T, line []byte, testCase []*testUnit) {
 }
 
 type testUnit struct {
-	Name string
-	Val  float64
+	Name       string
+	Val        float64
 	LabelValue string
 }
 
@@ -62,142 +62,142 @@ func (t *testUnit) key() string {
 func TestHandleLineWithAction(t *testing.T) {
 	tests := []*testUnit{
 		&testUnit{
-			Name: "action_processed",
-			Val:  100000,
+			Name:       "action_processed",
+			Val:        100000,
 			LabelValue: "test_action",
 		},
 		&testUnit{
-			Name: "action_failed",
-			Val:  2,
+			Name:       "action_failed",
+			Val:        2,
 			LabelValue: "test_action",
 		},
 		&testUnit{
-			Name: "action_suspended",
-			Val:  1,
+			Name:       "action_suspended",
+			Val:        1,
 			LabelValue: "test_action",
 		},
 		&testUnit{
-			Name: "action_suspended_duration",
-			Val:  1000,
+			Name:       "action_suspended_duration",
+			Val:        1000,
 			LabelValue: "test_action",
 		},
 		&testUnit{
-			Name: "action_resumed",
-			Val:  1,
+			Name:       "action_resumed",
+			Val:        1,
 			LabelValue: "test_action",
 		},
 	}
 
-	actionLog := []byte(`{"name":"test_action","processed":100000,"failed":2,"suspended":1,"suspended.duration":1000,"resumed":1}`)
+	actionLog := []byte(`2017-08-30T08:10:04.786350+00:00 some-node.example.org rsyslogd-pstats: {"name":"test_action","processed":100000,"failed":2,"suspended":1,"suspended.duration":1000,"resumed":1}`)
 	testHelper(t, actionLog, tests)
 }
 
 func TestHandleLineWithResource(t *testing.T) {
 	tests := []*testUnit{
 		&testUnit{
-			Name: "resource_utime",
-			Val:  10,
+			Name:       "resource_utime",
+			Val:        10,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_stime",
-			Val:  20,
+			Name:       "resource_stime",
+			Val:        20,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_maxrss",
-			Val:  30,
+			Name:       "resource_maxrss",
+			Val:        30,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_minflt",
-			Val:  40,
+			Name:       "resource_minflt",
+			Val:        40,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_majflt",
-			Val:  50,
+			Name:       "resource_majflt",
+			Val:        50,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_inblock",
-			Val:  60,
+			Name:       "resource_inblock",
+			Val:        60,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_oublock",
-			Val:  70,
+			Name:       "resource_oublock",
+			Val:        70,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_nvcsw",
-			Val:  80,
+			Name:       "resource_nvcsw",
+			Val:        80,
 			LabelValue: "resource-usage",
 		},
 		&testUnit{
-			Name: "resource_nivcsw",
-			Val:  90,
+			Name:       "resource_nivcsw",
+			Val:        90,
 			LabelValue: "resource-usage",
 		},
 	}
 
-	resourceLog := []byte(`{"name":"resource-usage","utime":10,"stime":20,"maxrss":30,"minflt":40,"majflt":50,"inblock":60,"oublock":70,"nvcsw":80,"nivcsw":90}`)
+	resourceLog := []byte(`2017-08-30T08:10:04.786350+00:00 some-node.example.org rsyslogd-pstats: {"name":"resource-usage","utime":10,"stime":20,"maxrss":30,"minflt":40,"majflt":50,"inblock":60,"oublock":70,"nvcsw":80,"nivcsw":90}`)
 	testHelper(t, resourceLog, tests)
 }
 
 func TestHandleLineWithInput(t *testing.T) {
 	tests := []*testUnit{
 		&testUnit{
-			Name: "input_submitted",
-			Val:  1000,
+			Name:       "input_submitted",
+			Val:        1000,
 			LabelValue: "test_input",
 		},
 	}
 
-	inputLog := []byte(`{"name":"test_input", "origin":"imuxsock", "submitted":1000}`)
+	inputLog := []byte(`2017-08-30T08:10:04.786350+00:00 some-node.example.org rsyslogd-pstats: {"name":"test_input", "origin":"imuxsock", "submitted":1000}`)
 	testHelper(t, inputLog, tests)
 }
 
 func TestHandleLineWithQueue(t *testing.T) {
 	tests := []*testUnit{
 		&testUnit{
-			Name: "queue_size",
-			Val:  10,
+			Name:       "queue_size",
+			Val:        10,
 			LabelValue: "main Q",
 		},
 		&testUnit{
-			Name: "queue_enqueued",
-			Val:  20,
+			Name:       "queue_enqueued",
+			Val:        20,
 			LabelValue: "main Q",
 		},
 		&testUnit{
-			Name: "queue_full",
-			Val:  30,
+			Name:       "queue_full",
+			Val:        30,
 			LabelValue: "main Q",
 		},
 		&testUnit{
-			Name: "queue_discarded_full",
-			Val:  40,
+			Name:       "queue_discarded_full",
+			Val:        40,
 			LabelValue: "main Q",
 		},
 		&testUnit{
-			Name: "queue_discarded_not_full",
-			Val:  50,
+			Name:       "queue_discarded_not_full",
+			Val:        50,
 			LabelValue: "main Q",
 		},
 		&testUnit{
-			Name: "queue_max_size",
-			Val:  60,
+			Name:       "queue_max_size",
+			Val:        60,
 			LabelValue: "main Q",
 		},
 	}
 
-	queueLog = []byte(`{"name":"main Q","size":10,"enqueued":20,"full":30,"discarded.full":40,"discarded.nf":50,"maxqsize":60}`)
+	queueLog := []byte(`2017-08-30T08:10:04.786350+00:00 some-node.example.org rsyslogd-pstats: {"name":"main Q","size":10,"enqueued":20,"full":30,"discarded.full":40,"discarded.nf":50,"maxqsize":60}`)
 	testHelper(t, queueLog, tests)
 }
 
 func TestHandleUnknown(t *testing.T) {
-	unknownLog := []byte(`{"a":"b"}`)
+	unknownLog := []byte(`2017-08-30T08:10:04.786350+00:00 some-node.example.org rsyslogd-pstats: {"a":"b"}`)
 
 	exporter := newRsyslogExporter()
 	exporter.handleStatLine(unknownLog)

diff --git a/queues_test.go b/queues_test.go
@@ -3,16 +3,16 @@ package main
 import "testing"
 
 var (
-	queueLog = []byte(`{"name":"main Q","size":10,"enqueued":20,"full":30,"discarded.full":40,"discarded.nf":50,"maxqsize":60}`)
+	queueStat = []byte(`{"name":"main Q","size":10,"enqueued":20,"full":30,"discarded.full":40,"discarded.nf":50,"maxqsize":60}`)
 )
 
 func TestNewQueueFromJSON(t *testing.T) {
-	logType := getStatType(queueLog)
+	logType := getStatType(queueStat)
 	if logType != rsyslogQueue {
 		t.Errorf("detected pstat type should be %d but is %d", rsyslogQueue, logType)
 	}
 
-	pstat, err := newQueueFromJSON([]byte(queueLog))
+	pstat, err := newQueueFromJSON([]byte(queueStat))
 	if err != nil {
 		t.Fatalf("expected parsing queue stat not to fail, got: %v", err)
 	}
@@ -47,7 +47,7 @@ func TestNewQueueFromJSON(t *testing.T) {
 }
 
 func TestQueueToPoints(t *testing.T) {
-	pstat, err := newQueueFromJSON([]byte(queueLog))
+	pstat, err := newQueueFromJSON([]byte(queueStat))
 	if err != nil {
 		t.Fatalf("expected parsing queue stat not to fail, got: %v", err)
 	}