Add error count prom metric (#14670)

* add error count prom metric * address review comments * add comment for response writer * update changelog
prysmaticlabs · Nov 27, 2024 · b28b1ed · b28b1ed
1 parent 74bb082
commit b28b1ed
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - PeerDAS: Added proto for `DataColumnIdentifier`, `DataColumnSidecar`, `DataColumnSidecarsByRangeRequest` and `MetadataV2`.
 - Better attestation packing for Electra. [PR](https://github.com/prysmaticlabs/prysm/pull/14534)
 - P2P: Add logs when a peer is (dis)connected. Add the reason of the disconnection when we initiate it.
+- Added a Prometheus error counter metric for HTTP requests to track beacon node requests.
 
 ### Changed
 

diff --git a/beacon-chain/rpc/BUILD.bazel b/beacon-chain/rpc/BUILD.bazel
@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "endpoints.go",
         "log.go",
+        "metrics.go",
         "service.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/rpc",

diff --git a/beacon-chain/rpc/endpoints.go b/beacon-chain/rpc/endpoints.go
@@ -34,18 +34,41 @@ type endpoint struct {
 	methods    []string
 }
 
+// responseWriter is the wrapper to http Response writer.
+type responseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+// WriteHeader wraps the WriteHeader method of the underlying http.ResponseWriter to capture the status code.
+// Refer for WriteHeader doc: https://pkg.go.dev/net/http@go1.23.3#ResponseWriter.
+func (w *responseWriter) WriteHeader(statusCode int) {
+	w.statusCode = statusCode
+	w.ResponseWriter.WriteHeader(statusCode)
+}
+
 func (e *endpoint) handlerWithMiddleware() http.HandlerFunc {
 	handler := http.Handler(e.handler)
 	for _, m := range e.middleware {
 		handler = m(handler)
 	}
-	return promhttp.InstrumentHandlerDuration(
+
+	handler = promhttp.InstrumentHandlerDuration(
 		httpRequestLatency.MustCurryWith(prometheus.Labels{"endpoint": e.name}),
 		promhttp.InstrumentHandlerCounter(
 			httpRequestCount.MustCurryWith(prometheus.Labels{"endpoint": e.name}),
 			handler,
 		),
 	)
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		rw := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
+		handler.ServeHTTP(rw, r)
+
+		if rw.statusCode >= 400 {
+			httpErrorCount.WithLabelValues(r.URL.Path, http.StatusText(rw.statusCode), r.Method).Inc()
+		}
+	}
 }
 
 func (s *Service) endpoints(

diff --git a/beacon-chain/rpc/metrics.go b/beacon-chain/rpc/metrics.go
@@ -0,0 +1,31 @@
+package rpc
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	httpRequestLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "http_request_latency_seconds",
+			Help:    "Latency of HTTP requests in seconds",
+			Buckets: []float64{0.001, 0.01, 0.025, 0.1, 0.25, 1, 2.5, 10},
+		},
+		[]string{"endpoint", "code", "method"},
+	)
+	httpRequestCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_request_count",
+			Help: "Number of HTTP requests",
+		},
+		[]string{"endpoint", "code", "method"},
+	)
+	httpErrorCount = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "http_error_count",
+			Help: "Total HTTP errors for beacon node requests",
+		},
+		[]string{"endpoint", "code", "method"},
+	)
+)
diff --git a/beacon-chain/rpc/service.go b/beacon-chain/rpc/service.go
@@ -14,8 +14,6 @@ import (
 	grpcopentracing "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"
 	grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/blockchain"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/builder"
 	"github.com/prysmaticlabs/prysm/v5/beacon-chain/cache"
@@ -57,24 +55,6 @@ import (
 
 const attestationBufferSize = 100
 
-var (
-	httpRequestLatency = promauto.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "http_request_latency_seconds",
-			Help:    "Latency of HTTP requests in seconds",
-			Buckets: []float64{0.001, 0.01, 0.025, 0.1, 0.25, 1, 2.5, 10},
-		},
-		[]string{"endpoint", "code", "method"},
-	)
-	httpRequestCount = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "http_request_count",
-			Help: "Number of HTTP requests",
-		},
-		[]string{"endpoint", "code", "method"},
-	)
-)
-
 // Service defining an RPC server for a beacon node.
 type Service struct {
 	cfg                  *Config