From b1345b6750581da052b10fa7bbbb8a8b4e299452 Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Fri, 6 Sep 2024 13:55:32 +0530 Subject: [PATCH 1/2] fix: skipping label if it contains special symbol --- pkg/logql/log/parser.go | 22 ++++++++++++++++------ pkg/logql/log/parser_test.go | 10 +++++----- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pkg/logql/log/parser.go b/pkg/logql/log/parser.go index 9a5ae1395069..19550778c2e4 100644 --- a/pkg/logql/log/parser.go +++ b/pkg/logql/log/parser.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "strings" "unicode/utf8" "github.com/grafana/jsonparser" @@ -200,12 +201,16 @@ func unescapeJSONString(b []byte) string { return "" } res := string(bU) - // rune error is rejected by Prometheus - for _, r := range res { + + // rune error is rejected by Prometheus hence replacing them with space + removeInvalidUtf := func(r rune) rune { if r == utf8.RuneError { - return "" + return -1 } + return r } + res = strings.Map(removeInvalidUtf, res) + return res } @@ -339,10 +344,15 @@ func (l *LogfmtParser) Process(_ int64, line []byte, lbs *LabelsBuilder) ([]byte } val := l.dec.Value() - // the rune error replacement is rejected by Prometheus, so we skip it. - if bytes.ContainsRune(val, utf8.RuneError) { - val = nil + + // the rune error replacement is rejected by Prometheus hence skiping them. + removeInvalidUtf := func(r rune) rune { + if r == utf8.RuneError { + return -1 + } + return r } + val = bytes.Map(removeInvalidUtf, val) if !l.keepEmpty && len(val) == 0 { continue diff --git a/pkg/logql/log/parser_test.go b/pkg/logql/log/parser_test.go index 654e35bb13e7..b0f61564c56c 100644 --- a/pkg/logql/log/parser_test.go +++ b/pkg/logql/log/parser_test.go @@ -79,11 +79,11 @@ func Test_jsonParser_Parse(t *testing.T) { }, { "utf8 error rune", - []byte(`{"counter":1,"foo":"�", "price": {"_net_":5.56909}}`), + []byte(`{"counter":1,"foo":"�f", "price": {"_net_":5.56909}}`), labels.EmptyLabels(), labels.FromStrings("counter", "1", "price__net_", "5.56909", - "foo", "", + "foo", "f", ), NoParserHints(), }, @@ -802,7 +802,7 @@ func TestLogfmtParser_parse(t *testing.T) { "utf8 error rune", []byte(`buzz=foo bar=�f`), labels.EmptyLabels(), - labels.FromStrings("buzz", "foo"), + labels.FromStrings("bar", "f", "buzz", "foo"), nil, NoParserHints(), }, @@ -1037,7 +1037,7 @@ func TestLogfmtParser_keepEmpty(t *testing.T) { false, labels.FromStrings("foo", "bar"), labels.FromStrings("foo", "bar", - "bar", "buzz"), + "bar", "buzz", "foo_extracted", "br"), }, { "utf8 error rune with keep empty", @@ -1045,7 +1045,7 @@ func TestLogfmtParser_keepEmpty(t *testing.T) { true, labels.FromStrings("foo", "bar"), labels.FromStrings("foo", "bar", - "foo_extracted", "", + "foo_extracted", "br", "bar", "buzz"), }, } From 6d7558685567702cda50e7eefadc78739a95aaff Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Mon, 9 Sep 2024 19:23:17 +0530 Subject: [PATCH 2/2] Check contains as per review comments --- pkg/logql/log/parser.go | 26 ++++++++++++-------------- pkg/logql/log/parser_test.go | 10 +++++----- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/pkg/logql/log/parser.go b/pkg/logql/log/parser.go index 19550778c2e4..c8e65061ba41 100644 --- a/pkg/logql/log/parser.go +++ b/pkg/logql/log/parser.go @@ -40,6 +40,14 @@ var ( errMissingCapture = errors.New("at least one named capture must be supplied") errFoundAllLabels = errors.New("found all required labels") errLabelDoesNotMatch = errors.New("found a label with a matcher that didn't match") + + // the rune error replacement is rejected by Prometheus hence replacing them with space. + removeInvalidUtf = func(r rune) rune { + if r == utf8.RuneError { + return 32 // rune value for space + } + return r + } ) type JSONParser struct { @@ -202,14 +210,9 @@ func unescapeJSONString(b []byte) string { } res := string(bU) - // rune error is rejected by Prometheus hence replacing them with space - removeInvalidUtf := func(r rune) rune { - if r == utf8.RuneError { - return -1 - } - return r + if strings.ContainsRune(res, utf8.RuneError) { + res = strings.Map(removeInvalidUtf, res) } - res = strings.Map(removeInvalidUtf, res) return res } @@ -345,14 +348,9 @@ func (l *LogfmtParser) Process(_ int64, line []byte, lbs *LabelsBuilder) ([]byte val := l.dec.Value() - // the rune error replacement is rejected by Prometheus hence skiping them. - removeInvalidUtf := func(r rune) rune { - if r == utf8.RuneError { - return -1 - } - return r + if bytes.ContainsRune(val, utf8.RuneError) { + val = bytes.Map(removeInvalidUtf, val) } - val = bytes.Map(removeInvalidUtf, val) if !l.keepEmpty && len(val) == 0 { continue diff --git a/pkg/logql/log/parser_test.go b/pkg/logql/log/parser_test.go index b0f61564c56c..28989b7cb5fe 100644 --- a/pkg/logql/log/parser_test.go +++ b/pkg/logql/log/parser_test.go @@ -79,11 +79,11 @@ func Test_jsonParser_Parse(t *testing.T) { }, { "utf8 error rune", - []byte(`{"counter":1,"foo":"�f", "price": {"_net_":5.56909}}`), + []byte(`{"counter":1,"foo":"�", "price": {"_net_":5.56909}}`), labels.EmptyLabels(), labels.FromStrings("counter", "1", "price__net_", "5.56909", - "foo", "f", + "foo", " ", ), NoParserHints(), }, @@ -802,7 +802,7 @@ func TestLogfmtParser_parse(t *testing.T) { "utf8 error rune", []byte(`buzz=foo bar=�f`), labels.EmptyLabels(), - labels.FromStrings("bar", "f", "buzz", "foo"), + labels.FromStrings("bar", " f", "buzz", "foo"), nil, NoParserHints(), }, @@ -1037,7 +1037,7 @@ func TestLogfmtParser_keepEmpty(t *testing.T) { false, labels.FromStrings("foo", "bar"), labels.FromStrings("foo", "bar", - "bar", "buzz", "foo_extracted", "br"), + "bar", "buzz", "foo_extracted", "b r"), }, { "utf8 error rune with keep empty", @@ -1045,7 +1045,7 @@ func TestLogfmtParser_keepEmpty(t *testing.T) { true, labels.FromStrings("foo", "bar"), labels.FromStrings("foo", "bar", - "foo_extracted", "br", + "foo_extracted", "b r", "bar", "buzz"), }, }