From fde0eb7f942202cb1f9efbd146f9abd60f6d3f97 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 9 May 2024 09:40:08 +0930 Subject: [PATCH] x-pack/filebeat/input/internal/httplog: improve req/resp logging (#39455) Attempt to log the request and response bodies and other details even when copying the body has been reported to have failed. (cherry picked from commit 3efb1e8612baae8b9b887afd0219c94caf36c7c1) --- CHANGELOG.next.asciidoc | 31 +++++++++++++++++++ .../input/internal/httplog/roundtripper.go | 26 +++++++--------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index caa91b893aac..55978417dd99 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -129,6 +129,37 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999] - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128] - Made GCS input GA and updated docs accordingly. {pull}37127[37127] +- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160] +- Prevent CEL input from re-entering the eval loop when an evaluation failed. {pull}37161[37161] +- Update CEL extensions library to v1.7.0. {pull}37172[37172] +- Add support for complete URL replacement in HTTPJSON chain steps. {pull}37486[37486] +- Add support for user-defined query selection in EntraID entity analytics provider. {pull}37653[37653] +- Update CEL extensions library to v1.8.0 to provide runtime error location reporting. {issue}37304[37304] {pull}37718[37718] +- Add request trace logging for chained API requests. {issue}37551[36551] {pull}37682[37682] +- Relax TCP/UDP metric polling expectations to improve metric collection. {pull}37714[37714] +- Add support for PEM-based Okta auth in HTTPJSON. {pull}37772[37772] +- Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] +- Added experimental version of the Websocket Input. {pull}37774[37774] +- Add support for PEM-based Okta auth in CEL. {pull}37813[37813] +- Add Salesforce input. {pull}37331[37331] +- Add ETW input. {pull}36915[36915] +- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] +- Add logging for cache processor file reads and writes. {pull}38052[38052] +- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] +- Support VPC endpoint for aws-s3 input SQS queue url. {pull}38189[38189] +- Improve rate limit handling by HTTPJSON {issue}36207[36207] {pull}38161[38161] {pull}38237[38237] +- Add parseDateInTZ value template for the HTTPJSON input. {pull}37738[37738] +- Add support for complex event objects in the HTTP Endpoint input. {issue}37910[37910] {pull}38193[38193] +- Parse more fields from Elasticsearch slowlogs {pull}38295[38295] +- Update CEL mito extensions to v1.10.0 to add base64 decode functions. {pull}38504[38504] +- Add support for Active Directory an entity analytics provider. {pull}37919[37919] +- Add AWS AWSHealth metricset. {pull}38370[38370] +- Add debugging breadcrumb to logs when writing request trace log. {pull}38636[38636] +- added benchmark input {pull}37437[37437] +- added benchmark input and discard output {pull}37437[37437] +- Ensure all responses sent by HTTP Endpoint are HTML-escaped. {pull}39329[39329] +- Update CEL mito extensions to v1.11.0 to improve type checking. {pull}39460[39460] +- Improve logging of request and response with request trace logging in error conditions. {pull}39455[39455] *Auditbeat* diff --git a/x-pack/filebeat/input/internal/httplog/roundtripper.go b/x-pack/filebeat/input/internal/httplog/roundtripper.go index eac54d7378f5..c313373fc304 100644 --- a/x-pack/filebeat/input/internal/httplog/roundtripper.go +++ b/x-pack/filebeat/input/internal/httplog/roundtripper.go @@ -109,14 +109,13 @@ func (rt *LoggingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err resp.Body, body, err = copyBody(resp.Body) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read response body: %s", err)) - } else { - respParts = append(respParts, - zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]), - zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)), - zap.Int("http.response.body.bytes", len(body)), - zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), - ) } + respParts = append(respParts, + zap.ByteString("http.response.body.content", body[:min(len(body), rt.maxBodyLen)]), + zap.Bool("http.response.body.truncated", rt.maxBodyLen < len(body)), + zap.Int("http.response.body.bytes", len(body)), + zap.String("http.response.mime_type", resp.Header.Get("Content-Type")), + ) message, err := httputil.DumpResponse(resp, false) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump response: %s", err)) @@ -178,14 +177,13 @@ func logRequest(log *zap.Logger, req *http.Request, maxBodyLen int, extra ...zap req.Body, body, err = copyBody(req.Body) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to read request body: %s", err)) - } else { - reqParts = append(reqParts, - zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]), - zap.Bool("http.request.body.truncated", maxBodyLen < len(body)), - zap.Int("http.request.body.bytes", len(body)), - zap.String("http.request.mime_type", req.Header.Get("Content-Type")), - ) } + reqParts = append(reqParts, + zap.ByteString("http.request.body.content", body[:min(len(body), maxBodyLen)]), + zap.Bool("http.request.body.truncated", maxBodyLen < len(body)), + zap.Int("http.request.body.bytes", len(body)), + zap.String("http.request.mime_type", req.Header.Get("Content-Type")), + ) message, err := httputil.DumpRequestOut(req, false) if err != nil { errorsMessages = append(errorsMessages, fmt.Sprintf("failed to dump request: %s", err))