Skip to content

Commit

Permalink
x-pack/filebeat/input/httpjson: Fix parseDate location offset (#37738)
Browse files Browse the repository at this point in the history
Add a new value template helper `parseDateInTZ` where users can apply the proper timezone when parsing dates, avoiding the limitations of `parseDate` with timezone abbreviations. It accepts numeric offsets and IANA time zone names.
  • Loading branch information
chemamartinez authored Feb 21, 2024
1 parent cdea924 commit 8214f9f
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
- Add ETW input. {pull}36915[36915]
- Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971]
- Add logging for cache processor file reads and writes. {pull}38052[38052]
- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738]

*Auditbeat*

Expand Down
3 changes: 2 additions & 1 deletion x-pack/filebeat/docs/inputs/input-httpjson.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,8 @@ Some built-in helper functions are provided to work with the input state inside
- `min`: returns the minimum of two values.
- `mul`: multiplies two integers.
- `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now.
- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.
- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Note: Parsing timezone abbreviations may cause ambiguities. Prefer `parseDateInTZ` for explicit timezone handling. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`.
- `parseDateInTZ`: parses a date string within a specified timezone (TZ), returning a `time.Time` in UTC. Specified timezone overwrites implicit timezone from the input date. Accepts timezone offsets ("-07:00", "-0700", "-07") or IANA Time Zone names ("America/New_York"). If TZ is invalid, defaults to UTC. Optional layout argument as in parseDate. Example: `[[ parseDateInTZ "2020-11-05T12:25:32" "America/New_York" ]]`, `[[ parseDateInTZ "2020-11-05T12:25:32" "-07:00" "RFC3339" ]]`.
- `parseDuration`: parses duration strings and returns `time.Duration`. Example: `[[parseDuration "1h"]]`.
- `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000]]` returns `2020-11-05 13:25:32 +0000 UTC`.
- `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000000000]]` returns `2020-11-05 13:25:32 +0000 UTC`.
Expand Down
53 changes: 53 additions & 0 deletions x-pack/filebeat/input/httpjson/value_tpl.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ func (t *valueTpl) Unpack(in string) error {
"mul": mul,
"now": now,
"parseDate": parseDate,
"parseDateInTZ": parseDateInTZ,
"parseDuration": parseDuration,
"parseTimestamp": parseTimestamp,
"parseTimestampMilli": parseTimestampMilli,
Expand Down Expand Up @@ -194,6 +195,58 @@ func parseDate(date string, layout ...string) time.Time {
return t.UTC()
}

// parseDateInTZ parses a date string within a specified timezone, returning a time.Time
// 'tz' is the timezone (offset or IANA name) for parsing
func parseDateInTZ(date string, tz string, layout ...string) time.Time {
var ly string
if len(layout) == 0 {
ly = defaultTimeLayout
} else {
ly = layout[0]
}
if found := predefinedLayouts[ly]; found != "" {
ly = found
}

var loc *time.Location
// Attempt to parse timezone as offset in various formats
for _, format := range []string{"-07", "-0700", "-07:00"} {
t, err := time.Parse(format, tz)
if err != nil {
continue
}
name, offset := t.Zone()
loc = time.FixedZone(name, offset)
break
}

// If parsing tz as offset fails, try loading location by name
if loc == nil {
var err error
loc, err = time.LoadLocation(tz)
if err != nil {
loc = time.UTC // Default to UTC on error
}
}

// Using Parse allows us not to worry about the timezone
// as the predefined timezone is applied afterwards
t, err := time.Parse(ly, date)
if err != nil {
return time.Time{}
}

// Manually create a new time object with the parsed date components and the desired location
// It allows interpreting the parsed time in the specified timezone
year, month, day := t.Date()
hour, min, sec := t.Clock()
nanosec := t.Nanosecond()
localTime := time.Date(year, month, day, hour, min, sec, nanosec, loc)

// Convert the time to UTC to standardize the output
return localTime.UTC()
}

func formatDate(date time.Time, layouttz ...string) string {
var layout, tz string
switch {
Expand Down
35 changes: 35 additions & 0 deletions x-pack/filebeat/input/httpjson/value_tpl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,41 @@ func TestValueTpl(t *testing.T) {
paramTr: transformable{},
expectedVal: "2020-11-05 12:25:32 +0000 UTC",
},
{
name: "func parseDateInTZ with RFC3339Nano and timezone offset",
value: `[[ parseDateInTZ "2020-11-05T12:25:32.1234567Z" "-0700" "RFC3339Nano" ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020-11-05 19:25:32.1234567 +0000 UTC",
},
{
name: "func parseDateInTZ defaults to RFC3339 with implicit offset and timezone",
value: `[[ parseDateInTZ "2020-11-05T12:25:32+04:00" "-0700" ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020-11-05 19:25:32 +0000 UTC",
},
{
name: "func parseDateInTZ defaults to RFC3339 with IANA timezone",
value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "America/New_York" ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020-11-05 17:25:32 +0000 UTC",
},
{
name: "func parseDateInTZ with custom layout and timezone name",
value: `[[ parseDateInTZ "Thu Nov 5 12:25:32 2020" "Europe/Paris" "Mon Jan _2 15:04:05 2006" ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020-11-05 11:25:32 +0000 UTC",
},
{
name: "func parseDateInTZ with invalid timezone",
value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "Invalid/Timezone" ]]`,
paramCtx: emptyTransformContext(),
paramTr: transformable{},
expectedVal: "2020-11-05 12:25:32 +0000 UTC",
},
{
name: "func formatDate",
setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } },
Expand Down

0 comments on commit 8214f9f

Please sign in to comment.