From 117e60583d4ec2e953174d49386fc4b1b80a7d0c Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 1 Apr 2024 22:06:48 +0400 Subject: [PATCH] feat: add support for static extra fields for JSON logs Fixes #7356 Signed-off-by: Andrey Smirnov --- hack/release.toml | 16 + .../pkg/controllers/runtime/kmsg_log.go | 23 +- .../pkg/runtime/logging/sender_jsonlines.go | 16 +- .../runtime/logging/sender_jsonlines_test.go | 336 ++++++++++++++++++ .../runtime/v1alpha2/v1alpha2_controller.go | 56 ++- pkg/machinery/config/config/machine.go | 1 + .../config/schemas/config.schema.json | 12 + .../config/types/v1alpha1/v1alpha1_logging.go | 5 + .../config/types/v1alpha1/v1alpha1_types.go | 3 + .../types/v1alpha1/v1alpha1_types_doc.go | 7 + .../types/v1alpha1/zz_generated.deepcopy.go | 7 + .../configuration/v1alpha1/config.md | 1 + .../content/v1.7/schemas/config.schema.json | 12 + .../talos-guides/configuration/logging.md | 14 + 14 files changed, 492 insertions(+), 17 deletions(-) create mode 100644 internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go diff --git a/hack/release.toml b/hack/release.toml index 62eec59f93..e8519fc840 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -177,6 +177,22 @@ kind: WatchdogTimerConfig device: /dev/watchdog0 timeout: 3m0s ``` +""" + + [notes.logging] + title = "Logging" + description = """\ +Talos Linux now supports setting extra tags when sending logs in JSON format: + +```yaml +machine: + logging: + destinations: + - endpoint: "udp://127.0.0.1:12345/" + format: "json_lines" + extraTags: + server: s03-rack07 +``` """ [make_deps] diff --git a/internal/app/machined/pkg/controllers/runtime/kmsg_log.go b/internal/app/machined/pkg/controllers/runtime/kmsg_log.go index 59faad8f88..0c28bdd86d 100644 --- a/internal/app/machined/pkg/controllers/runtime/kmsg_log.go +++ b/internal/app/machined/pkg/controllers/runtime/kmsg_log.go @@ -23,6 +23,8 @@ import ( networkutils "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/network/utils" machinedruntime "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/logging" + "github.com/siderolabs/talos/pkg/machinery/config/config" + "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/resources/network" "github.com/siderolabs/talos/pkg/machinery/resources/runtime" ) @@ -110,6 +112,22 @@ func (ctrl *KmsgLogDeliveryController) Run(ctx context.Context, r controller.Run } } +type logConfig struct { + endpoint *url.URL +} + +func (c logConfig) Format() string { + return constants.LoggingFormatJSONLines +} + +func (c logConfig) Endpoint() *url.URL { + return c.endpoint +} + +func (c logConfig) ExtraTags() map[string]string { + return nil +} + //nolint:gocyclo func (ctrl *KmsgLogDeliveryController) deliverLogs(ctx context.Context, r controller.Runtime, logger *zap.Logger, kmsgCh <-chan kmsg.Packet, destURLs []*url.URL) error { if ctrl.drainSub == nil { @@ -117,7 +135,10 @@ func (ctrl *KmsgLogDeliveryController) deliverLogs(ctx context.Context, r contro } // initialize all log senders - senders := xslices.Map(destURLs, logging.NewJSONLines) + destLogConfigs := xslices.Map(destURLs, func(u *url.URL) config.LoggingDestination { + return logConfig{endpoint: u} + }) + senders := xslices.Map(destLogConfigs, logging.NewJSONLines) defer func() { closeCtx, closeCtxCancel := context.WithTimeout(context.Background(), logCloseTimeout) diff --git a/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go b/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go index 7878f50a00..5e87ba91bd 100644 --- a/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go +++ b/internal/app/machined/pkg/runtime/logging/sender_jsonlines.go @@ -13,10 +13,12 @@ import ( "time" "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/pkg/machinery/config/config" ) type jsonLinesSender struct { - endpoint *url.URL + endpoint *url.URL + extraTags map[string]string sema chan struct{} conn net.Conn @@ -24,13 +26,15 @@ type jsonLinesSender struct { // NewJSONLines returns log sender that sends logs in JSON over TCP (newline-delimited) // or UDP (one message per packet). -func NewJSONLines(endpoint *url.URL) runtime.LogSender { +func NewJSONLines(cfg config.LoggingDestination) runtime.LogSender { sema := make(chan struct{}, 1) sema <- struct{}{} return &jsonLinesSender{ - endpoint: endpoint, - sema: sema, + endpoint: cfg.Endpoint(), + extraTags: cfg.ExtraTags(), + + sema: sema, } } @@ -55,6 +59,10 @@ func (j *jsonLinesSender) marshalJSON(e *runtime.LogEvent) ([]byte, error) { m["talos-time"] = e.Time.Format(time.RFC3339Nano) m["talos-level"] = e.Level.String() + for k, v := range j.extraTags { + m[k] = v + } + return json.Marshal(m) } diff --git a/internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go b/internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go new file mode 100644 index 0000000000..40bcc8d045 --- /dev/null +++ b/internal/app/machined/pkg/runtime/logging/sender_jsonlines_test.go @@ -0,0 +1,336 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package logging_test + +import ( + "bufio" + "context" + "encoding/json" + "net" + "net/url" + "sync" + "testing" + "time" + + "github.com/siderolabs/gen/channel" + "github.com/siderolabs/gen/ensure" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/app/machined/pkg/runtime/logging" + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func udpHandler(ctx context.Context, t *testing.T, conn net.PacketConn, sendCh chan<- []byte) { + t.Helper() + + for { + select { + case <-ctx.Done(): + return + default: + } + + if err := conn.SetReadDeadline(time.Now().Add(10 * time.Millisecond)); err != nil { + t.Logf("failed to set read deadline: %v", err) + + return + } + + buf := make([]byte, 1024) + + n, _, err := conn.ReadFrom(buf) + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + continue + } + + t.Logf("failed to read from UDP connection: %v", err) + + return + } + + if !channel.SendWithContext(ctx, sendCh, buf[:n]) { + return + } + } +} + +func tcpHandler(ctx context.Context, t *testing.T, conn net.Listener, sendCh chan<- []byte) { + t.Helper() + + for { + select { + case <-ctx.Done(): + return + default: + } + + if err := conn.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Millisecond)); err != nil { + t.Logf("failed to set accept deadline: %v", err) + + return + } + + c, err := conn.Accept() + if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + continue + } + + t.Logf("failed to accept UDP connection: %v", err) + + return + } + + go func() { + defer c.Close() //nolint:errcheck + + scanner := bufio.NewScanner(c) + + for scanner.Scan() { + if !channel.SendWithContext(ctx, sendCh, scanner.Bytes()) { + return + } + } + }() + } +} + +type loggingDestination struct { + endpoint *url.URL + extraTags map[string]string +} + +func (l *loggingDestination) Endpoint() *url.URL { + return l.endpoint +} + +func (l *loggingDestination) ExtraTags() map[string]string { + return l.extraTags +} + +func (l *loggingDestination) Format() string { + return constants.LoggingFormatJSONLines +} + +func TestSenderJSONLines(t *testing.T) { //nolint:tparallel + t.Parallel() + + lisUDP, err := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, lisUDP.Close()) + }) + + lisTCP, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, lisTCP.Close()) + }) + + udpEndpoint := lisUDP.LocalAddr().String() + tcpEndpoint := lisTCP.Addr().String() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + + sendCh := make(chan []byte, 32) + + var wg sync.WaitGroup + + wg.Add(1) + + go func() { + defer wg.Done() + + udpHandler(ctx, t, lisUDP, sendCh) + }() + + wg.Add(1) + + go func() { + defer wg.Done() + + tcpHandler(ctx, t, lisTCP, sendCh) + }() + + t.Cleanup(wg.Wait) + + for _, test := range []struct { + name string + + endpoint *url.URL + extraTags map[string]string + + messages []*runtime.LogEvent + + expected []map[string]any + }{ + { + name: "UDP", + + endpoint: ensure.Value(url.Parse("udp://" + udpEndpoint)), + + messages: []*runtime.LogEvent{ + { + Msg: "msg1", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + { + Msg: "msg2", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:01Z")), + Level: zapcore.DebugLevel, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "msg": "msg1", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + { + "msg": "msg2", + "talos-level": "debug", + "talos-time": "2021-01-01T00:00:01Z", + }, + }, + }, + { + name: "UDP with extra tags", + + endpoint: ensure.Value(url.Parse("udp://" + udpEndpoint)), + extraTags: map[string]string{ + "extra1": "value1", + }, + + messages: []*runtime.LogEvent{ + { + Msg: "msg1", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + { + Msg: "msg2", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:01Z")), + Level: zapcore.DebugLevel, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "extra1": "value1", + "msg": "msg1", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + { + "msg": "msg2", + "extra1": "value1", + "talos-level": "debug", + "talos-time": "2021-01-01T00:00:01Z", + }, + }, + }, + { + name: "TCP", + + endpoint: ensure.Value(url.Parse("tcp://" + tcpEndpoint)), + + messages: []*runtime.LogEvent{ + { + Msg: "hello", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "msg": "hello", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + }, + }, + { + name: "TCP with extra tags", + + endpoint: ensure.Value(url.Parse("tcp://" + tcpEndpoint)), + extraTags: map[string]string{ + "extra1": "value1", + }, + + messages: []*runtime.LogEvent{ + { + Msg: "hello", + Time: ensure.Value(time.Parse(time.RFC3339Nano, "2021-01-01T00:00:00Z")), + Level: zapcore.InfoLevel, + Fields: map[string]any{ + "field1": "value1", + }, + }, + }, + + expected: []map[string]any{ + { + "field1": "value1", + "extra1": "value1", + "msg": "hello", + "talos-level": "info", + "talos-time": "2021-01-01T00:00:00Z", + }, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + // not parallel - need sequential execution + loggingCfg := &loggingDestination{ + endpoint: test.endpoint, + extraTags: test.extraTags, + } + + sender := logging.NewJSONLines(loggingCfg) + + for _, msg := range test.messages { + require.NoError(t, sender.Send(ctx, msg)) + } + + for _, expected := range test.expected { + select { + case <-time.After(time.Second): + t.Fatalf("timed out waiting for message") + case msg := <-sendCh: + var m map[string]any + + require.NoError(t, json.Unmarshal(msg, &m)) + + require.Equal(t, expected, m) + } + } + + require.NoError(t, sender.Close(ctx)) + }) + } + + cancel() +} diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index cc6f7aa356..c2a0ee9c63 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -352,6 +352,34 @@ func (ctrl *Controller) DependencyGraph() (*controller.DependencyGraph, error) { return ctrl.controllerRuntime.GetDependencyGraph() } +type loggingDestination struct { + Format string + Endpoint *url.URL + ExtraTags map[string]string +} + +func (a *loggingDestination) Equal(b *loggingDestination) bool { + if a.Format != b.Format { + return false + } + + if a.Endpoint.String() != b.Endpoint.String() { + return false + } + + if len(a.ExtraTags) != len(b.ExtraTags) { + return false + } + + for k, v := range a.ExtraTags { + if vv, ok := b.ExtraTags[k]; !ok || vv != v { + return false + } + } + + return true +} + func (ctrl *Controller) watchMachineConfig(ctx context.Context) { watchCh := make(chan state.Event) @@ -365,7 +393,7 @@ func (ctrl *Controller) watchMachineConfig(ctx context.Context) { return } - var loggingEndpoints []*url.URL + var loggingDestinations []loggingDestination for { var cfg talosconfig.Config @@ -384,9 +412,9 @@ func (ctrl *Controller) watchMachineConfig(ctx context.Context) { ctrl.updateConsoleLoggingConfig(cfg.Debug()) if cfg.Machine() == nil { - ctrl.updateLoggingConfig(ctx, nil, &loggingEndpoints) + ctrl.updateLoggingConfig(ctx, nil, &loggingDestinations) } else { - ctrl.updateLoggingConfig(ctx, cfg.Machine().Logging().Destinations(), &loggingEndpoints) + ctrl.updateLoggingConfig(ctx, cfg.Machine().Logging().Destinations(), &loggingDestinations) } } } @@ -403,23 +431,27 @@ func (ctrl *Controller) updateConsoleLoggingConfig(debug bool) { } } -func (ctrl *Controller) updateLoggingConfig(ctx context.Context, dests []talosconfig.LoggingDestination, prevLoggingEndpoints *[]*url.URL) { - loggingEndpoints := make([]*url.URL, len(dests)) +func (ctrl *Controller) updateLoggingConfig(ctx context.Context, dests []talosconfig.LoggingDestination, prevLoggingDestinations *[]loggingDestination) { + loggingDestinations := make([]loggingDestination, len(dests)) for i, dest := range dests { switch f := dest.Format(); f { case constants.LoggingFormatJSONLines: - loggingEndpoints[i] = dest.Endpoint() + loggingDestinations[i] = loggingDestination{ + Format: f, + Endpoint: dest.Endpoint(), + ExtraTags: dest.ExtraTags(), + } default: // should not be possible due to validation panic(fmt.Sprintf("unhandled log destination format %q", f)) } } - loggingChanged := len(*prevLoggingEndpoints) != len(loggingEndpoints) + loggingChanged := len(*prevLoggingDestinations) != len(loggingDestinations) if !loggingChanged { - for i, u := range *prevLoggingEndpoints { - if u.String() != loggingEndpoints[i].String() { + for i, u := range *prevLoggingDestinations { + if !u.Equal(&loggingDestinations[i]) { loggingChanged = true break @@ -431,12 +463,12 @@ func (ctrl *Controller) updateLoggingConfig(ctx context.Context, dests []talosco return } - *prevLoggingEndpoints = loggingEndpoints + *prevLoggingDestinations = loggingDestinations var prevSenders []runtime.LogSender - if len(loggingEndpoints) > 0 { - senders := xslices.Map(loggingEndpoints, runtimelogging.NewJSONLines) + if len(loggingDestinations) > 0 { + senders := xslices.Map(dests, runtimelogging.NewJSONLines) ctrl.logger.Info("enabling JSON logging") prevSenders = ctrl.loggingManager.SetSenders(senders) diff --git a/pkg/machinery/config/config/machine.go b/pkg/machinery/config/config/machine.go index 3e1ee16d21..d6ff0ff84a 100644 --- a/pkg/machinery/config/config/machine.go +++ b/pkg/machinery/config/config/machine.go @@ -445,6 +445,7 @@ type Logging interface { // LoggingDestination describes logging destination. type LoggingDestination interface { Endpoint() *url.URL + ExtraTags() map[string]string Format() string } diff --git a/pkg/machinery/config/schemas/config.schema.json b/pkg/machinery/config/schemas/config.schema.json index 1fe9d658ca..ed67cc2e27 100644 --- a/pkg/machinery/config/schemas/config.schema.json +++ b/pkg/machinery/config/schemas/config.schema.json @@ -2299,6 +2299,18 @@ "description": "Logs format.\n", "markdownDescription": "Logs format.", "x-intellij-html-description": "\u003cp\u003eLogs format.\u003c/p\u003e\n" + }, + "extraTags": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object", + "title": "extraTags", + "description": "Extra tags (key-value) pairs to attach to every log message sent.\n", + "markdownDescription": "Extra tags (key-value) pairs to attach to every log message sent.", + "x-intellij-html-description": "\u003cp\u003eExtra tags (key-value) pairs to attach to every log message sent.\u003c/p\u003e\n" } }, "additionalProperties": false, diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_logging.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_logging.go index de05589baa..cc3a8dfa98 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_logging.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_logging.go @@ -59,6 +59,11 @@ func (ld LoggingDestination) Endpoint() *url.URL { return ld.LoggingEndpoint.URL } +// ExtraTags implements config.LoggingDestination interface. +func (ld LoggingDestination) ExtraTags() map[string]string { + return ld.LoggingExtraTags +} + // Format implements config.LoggingDestination interface. func (ld LoggingDestination) Format() string { return ld.LoggingFormat diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go index d7a0a0ee91..4ffa4e1999 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go @@ -2392,6 +2392,9 @@ type LoggingDestination struct { // values: // - json_lines LoggingFormat string `yaml:"format"` + // description: | + // Extra tags (key-value) pairs to attach to every log message sent. + LoggingExtraTags map[string]string `yaml:"extraTags,omitempty"` } // KernelConfig struct configures Talos Linux kernel. diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go index 07bc0e9ff8..e21b69256a 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go @@ -3906,6 +3906,13 @@ func (LoggingDestination) Doc() *encoder.Doc { "json_lines", }, }, + { + Name: "extraTags", + Type: "map[string]string", + Note: "", + Description: "Extra tags (key-value) pairs to attach to every log message sent.", + Comments: [3]string{"" /* encoder.HeadComment */, "Extra tags (key-value) pairs to attach to every log message sent." /* encoder.LineComment */, "" /* encoder.FootComment */}, + }, }, } diff --git a/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go b/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go index 2bcd9c7893..e4772b8e68 100644 --- a/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go @@ -1503,6 +1503,13 @@ func (in *LoggingDestination) DeepCopyInto(out *LoggingDestination) { in, out := &in.LoggingEndpoint, &out.LoggingEndpoint *out = (*in).DeepCopy() } + if in.LoggingExtraTags != nil { + in, out := &in.LoggingExtraTags, &out.LoggingExtraTags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/website/content/v1.7/reference/configuration/v1alpha1/config.md b/website/content/v1.7/reference/configuration/v1alpha1/config.md index fa295ff160..7987bee6a5 100644 --- a/website/content/v1.7/reference/configuration/v1alpha1/config.md +++ b/website/content/v1.7/reference/configuration/v1alpha1/config.md @@ -2667,6 +2667,7 @@ endpoint: udp://127.0.0.1:12345 endpoint: tcp://1.2.3.4:12345 {{< /highlight >}} | | |`format` |string |Logs format. |`json_lines`
| +|`extraTags` |map[string]string |Extra tags (key-value) pairs to attach to every log message sent. | | diff --git a/website/content/v1.7/schemas/config.schema.json b/website/content/v1.7/schemas/config.schema.json index 1fe9d658ca..ed67cc2e27 100644 --- a/website/content/v1.7/schemas/config.schema.json +++ b/website/content/v1.7/schemas/config.schema.json @@ -2299,6 +2299,18 @@ "description": "Logs format.\n", "markdownDescription": "Logs format.", "x-intellij-html-description": "\u003cp\u003eLogs format.\u003c/p\u003e\n" + }, + "extraTags": { + "patternProperties": { + ".*": { + "type": "string" + } + }, + "type": "object", + "title": "extraTags", + "description": "Extra tags (key-value) pairs to attach to every log message sent.\n", + "markdownDescription": "Extra tags (key-value) pairs to attach to every log message sent.", + "x-intellij-html-description": "\u003cp\u003eExtra tags (key-value) pairs to attach to every log message sent.\u003c/p\u003e\n" } }, "additionalProperties": false, diff --git a/website/content/v1.7/talos-guides/configuration/logging.md b/website/content/v1.7/talos-guides/configuration/logging.md index b061738a7e..9eb7a1d9d1 100644 --- a/website/content/v1.7/talos-guides/configuration/logging.md +++ b/website/content/v1.7/talos-guides/configuration/logging.md @@ -89,6 +89,20 @@ Messages are newline-separated when sent over TCP. Over UDP messages are sent with one message per packet. `msg`, `talos-level`, `talos-service`, and `talos-time` fields are always present; there may be additional fields. +Every message sent can be enhanced with additional fields by using the `extraTags` field in the machine configuration: + +```yaml +machine: + logging: + destinations: + - endpoint: "udp://127.0.0.1:12345/" + format: "json_lines" + extraTags: + server: s03-rack07 +``` + +The specified `extraTags` are added to every message sent to the destination verbatim. + ### Kernel logs Kernel log delivery can be enabled with the `talos.logging.kernel` kernel command line argument, which can be specified