Skip to content

Commit

Permalink
chore(observability): set source fields to mean service (vectordotdev…
Browse files Browse the repository at this point in the history
…#17470)

This adds the `service` meaning to the `appname` field from syslog,
`app_name` from `heroku_logs`, `source` from `splunk_hec`.

It also adds a new field to `demo_logs` called `service` and populates
it with the value `vector`.

The `datadog_agent` source already [handles
this](https://github.com/vectordotdev/vector/blob/master/src/sources/datadog_agent/mod.rs#L225-L231).

I can't think of any other sources that may specify a potential service
field, but I am very open to suggestions!

---------

Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com>
  • Loading branch information
StephenWakely authored May 25, 2023
1 parent 426d660 commit 670bdea
Show file tree
Hide file tree
Showing 6 changed files with 58 additions and 15 deletions.
14 changes: 11 additions & 3 deletions lib/codecs/src/decoding/format/syslog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,11 @@ impl SyslogDeserializerConfig {
)
.optional_field(&owned_value_path!("facility"), Kind::bytes(), None)
.optional_field(&owned_value_path!("version"), Kind::integer(), None)
.optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("appname"),
Kind::bytes(),
Some("service"),
)
.optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("procid"),
Expand Down Expand Up @@ -112,7 +116,11 @@ impl SyslogDeserializerConfig {
)
.optional_field(&owned_value_path!("facility"), Kind::bytes(), None)
.optional_field(&owned_value_path!("version"), Kind::integer(), None)
.optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("appname"),
Kind::bytes(),
Some("service"),
)
.optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("procid"),
Expand Down Expand Up @@ -172,7 +180,7 @@ impl SyslogDeserializerConfig {
None,
&owned_value_path!("appname"),
Kind::bytes().or_undefined(),
None,
Some("service"),
)
.with_source_metadata(
source,
Expand Down
12 changes: 10 additions & 2 deletions src/sources/datadog_agent/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1690,7 +1690,11 @@ fn test_config_outputs() {
)
.optional_field(&owned_value_path!("facility"), Kind::bytes(), None)
.optional_field(&owned_value_path!("version"), Kind::integer(), None)
.optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("appname"),
Kind::bytes(),
Some("service"),
)
.optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("procid"),
Expand Down Expand Up @@ -1766,7 +1770,11 @@ fn test_config_outputs() {
Kind::integer(),
None,
)
.optional_field(&owned_value_path!("appname"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("appname"),
Kind::bytes(),
Some("service"),
)
.optional_field(&owned_value_path!("msgid"), Kind::bytes(), None)
.optional_field(
&owned_value_path!("procid"),
Expand Down
23 changes: 21 additions & 2 deletions src/sources/demo_logs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use codecs::{
};
use fakedata::logs::*;
use futures::StreamExt;
use lookup::{owned_value_path, path};
use rand::seq::SliceRandom;
use serde_with::serde_as;
use snafu::Snafu;
Expand All @@ -15,7 +16,11 @@ use vector_common::internal_event::{
ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol,
};
use vector_config::configurable_component;
use vector_core::{config::LogNamespace, EstimatedJsonEncodedSizeOf};
use vector_core::{
config::{LegacyKey, LogNamespace},
EstimatedJsonEncodedSizeOf,
};
use vrl::value::Kind;

use crate::{
codecs::{Decoder, DecodingConfig},
Expand Down Expand Up @@ -249,6 +254,13 @@ async fn demo_logs_source(
DemoLogsConfig::NAME,
now,
);
log_namespace.insert_source_metadata(
"service",
log,
Some(LegacyKey::InsertIfEmpty(path!("service"))),
path!("service"),
"vector",
);

event
});
Expand Down Expand Up @@ -300,7 +312,14 @@ impl SourceConfig for DemoLogsConfig {
let schema_definition = self
.decoding
.schema_definition(log_namespace)
.with_standard_vector_source_metadata();
.with_standard_vector_source_metadata()
.with_source_metadata(
DemoLogsConfig::NAME,
Some(LegacyKey::InsertIfEmpty(owned_value_path!("service"))),
&owned_value_path!("service"),
Kind::bytes(),
Some("service"),
);

vec![SourceOutput::new_logs(
self.decoding.output_type(),
Expand Down
10 changes: 7 additions & 3 deletions src/sources/heroku_logs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ impl LogplexConfig {
Some(LegacyKey::InsertIfEmpty(owned_value_path!("app_name"))),
&owned_value_path!("app_name"),
Kind::bytes(),
None,
Some("service"),
)
.with_source_metadata(
LogplexConfig::NAME,
Expand Down Expand Up @@ -694,7 +694,7 @@ mod tests {
.with_metadata_field(
&owned_value_path!(LogplexConfig::NAME, "app_name"),
Kind::bytes(),
None,
Some("service"),
)
.with_metadata_field(
&owned_value_path!(LogplexConfig::NAME, "proc_id"),
Expand Down Expand Up @@ -731,7 +731,11 @@ mod tests {
.with_event_field(&owned_value_path!("source_type"), Kind::bytes(), None)
.with_event_field(&owned_value_path!("timestamp"), Kind::timestamp(), None)
.with_event_field(&owned_value_path!("host"), Kind::bytes(), Some("host"))
.with_event_field(&owned_value_path!("app_name"), Kind::bytes(), None)
.with_event_field(
&owned_value_path!("app_name"),
Kind::bytes(),
Some("service"),
)
.with_event_field(&owned_value_path!("proc_id"), Kind::bytes(), None)
.unknown_fields(Kind::bytes());

Expand Down
10 changes: 7 additions & 3 deletions src/sources/splunk_hec/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ impl SourceConfig for SplunkConfig {
Some(LegacyKey::Overwrite(owned_value_path!(SOURCE))),
&owned_value_path!("source"),
Kind::bytes(),
None,
Some("service"),
)
// Not to be confused with `source_type`.
.with_source_metadata(
Expand Down Expand Up @@ -2475,7 +2475,7 @@ mod tests {
.with_metadata_field(
&owned_value_path!("splunk_hec", "source"),
Kind::bytes(),
None,
Some("service"),
)
.with_metadata_field(
&owned_value_path!("splunk_hec", "channel"),
Expand Down Expand Up @@ -2519,7 +2519,11 @@ mod tests {
.with_event_field(&owned_value_path!("source_type"), Kind::bytes(), None)
.with_event_field(&owned_value_path!("splunk_channel"), Kind::bytes(), None)
.with_event_field(&owned_value_path!("splunk_index"), Kind::bytes(), None)
.with_event_field(&owned_value_path!("splunk_source"), Kind::bytes(), None)
.with_event_field(
&owned_value_path!("splunk_source"),
Kind::bytes(),
Some("service"),
)
.with_event_field(&owned_value_path!("splunk_sourcetype"), Kind::bytes(), None)
.with_event_field(&owned_value_path!("timestamp"), Kind::timestamp(), None);

Expand Down
4 changes: 2 additions & 2 deletions src/sources/syslog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ mod test {
.with_metadata_field(
&owned_value_path!("syslog", "appname"),
Kind::bytes().or_undefined(),
None,
Some("service"),
)
.with_metadata_field(
&owned_value_path!("syslog", "msgid"),
Expand Down Expand Up @@ -628,7 +628,7 @@ mod test {
.with_event_field(
&owned_value_path!("appname"),
Kind::bytes().or_undefined(),
None,
Some("service"),
)
.with_event_field(
&owned_value_path!("msgid"),
Expand Down

0 comments on commit 670bdea

Please sign in to comment.