Detect body format after reading from cache and restore content-type …

…header (#3759)

* Detect body format after reading from cache and restore content-type header

* comment

* changelog

CHANGELOG.md

@@ -10,6 +10,7 @@
 * [ENHANCEMENT] Performance improvement for `rate() by ()` queries [#3719](https://github.com/grafana/tempo/pull/3719) (@mapno)
 * [ENHANCEMENT] Use multiple goroutines to unmarshal responses in parallel in the query frontend. [#3713](https://github.com/grafana/tempo/pull/3713) (@joe-elliott)
 * [BUGFIX] Fix metrics queries when grouping by attributes that may not exist [#3734](https://github.com/grafana/tempo/pull/3734) (@mdisibio)
+* [BUGFIX] Fix frontend parsing error on cached responses [#3759](https://github.com/grafana/tempo/pull/3759) (@mdisibio)
 * [BUGFIX] max_global_traces_per_user: take into account ingestion.tenant_shard_size when converting to local limit [#3618](https://github.com/grafana/tempo/pull/3618) (@kvrhdn)
 
 ## v2.5.0

modules/frontend/pipeline/sync_handler_cache.go

@@ -10,6 +10,7 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/gogo/protobuf/jsonpb"
 	"github.com/gogo/protobuf/proto"
+	"github.com/grafana/tempo/pkg/api"
 	"github.com/grafana/tempo/pkg/cache"
 )
 
@@ -39,11 +40,26 @@ func (c cachingWare) RoundTrip(req *http.Request) (*http.Response, error) {
 	if ok && len(key) > 0 {
 		body := c.cache.fetchBytes(key)
 		if len(body) > 0 {
-			return &http.Response{
+			resp := &http.Response{
+				Header:     http.Header{},
 				StatusCode: http.StatusOK,
 				Status:     http.StatusText(http.StatusOK),
 				Body:       io.NopCloser(bytes.NewBuffer(body)),
-			}, nil
+			}
+
+			// We aren't capturing the original content type in the cache, just the raw bytes.
+			// Detect it and readd it, so the upstream code can parse the body.
+			// TODO - Cache should capture all of the relevant parts of the
+			// original response including both content-type and content-length headers, possibly more.
+			// But upgrading the cache format requires migration/detection of previous format either way.
+			// It's tempting to use https://pkg.go.dev/net/http#DetectContentType but it doesn't detect
+			// json or proto.
+			if body[0] == '{' {
+				resp.Header.Add(api.HeaderContentType, api.HeaderAcceptJSON)
+			} else {
+				resp.Header.Add(api.HeaderContentType, api.HeaderAcceptProtobuf)
+			}
+			return resp, nil
 		}
 	}