diff --git a/lib/codecs/src/decoding/format/json.rs b/lib/codecs/src/decoding/format/json.rs index 32f28e5e58436..164d91756ff15 100644 --- a/lib/codecs/src/decoding/format/json.rs +++ b/lib/codecs/src/decoding/format/json.rs @@ -2,9 +2,11 @@ use std::convert::TryInto; use bytes::Bytes; use chrono::Utc; +use derivative::Derivative; use lookup::PathPrefix; use serde::{Deserialize, Serialize}; use smallvec::{smallvec, SmallVec}; +use vector_config::configurable_component; use vector_core::{ config::{log_schema, DataType, LogNamespace}, event::Event, @@ -16,7 +18,36 @@ use super::Deserializer; /// Config used to build a `JsonDeserializer`. #[derive(Debug, Clone, Default, Deserialize, Serialize)] -pub struct JsonDeserializerConfig; +pub struct JsonDeserializerConfig { + #[serde( + default, + skip_serializing_if = "vector_core::serde::skip_serializing_if_default" + )] + /// Options for the JSON deserializer. + pub json: JsonDeserializerOptions, +} + +/// JSON-specific decoding options. +#[configurable_component] +#[derive(Debug, Clone, PartialEq, Eq, Derivative)] +#[derivative(Default)] +pub struct JsonDeserializerOptions { + /// Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + /// + /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + /// + /// [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + #[serde( + default = "default_lossy", + skip_serializing_if = "vector_core::serde::skip_serializing_if_default" + )] + #[derivative(Default(value = "default_lossy()"))] + lossy: bool, +} + +const fn default_lossy() -> bool { + true +} impl JsonDeserializerConfig { /// Build the `JsonDeserializer` from this configuration. @@ -56,19 +87,23 @@ impl JsonDeserializerConfig { impl JsonDeserializerConfig { /// Creates a new `JsonDeserializerConfig`. - pub fn new() -> Self { - Default::default() + pub fn new(options: JsonDeserializerOptions) -> Self { + Self { json: options } } } /// Deserializer that builds `Event`s from a byte frame containing JSON. -#[derive(Debug, Clone, Default)] -pub struct JsonDeserializer; +#[derive(Debug, Clone, Derivative)] +#[derivative(Default)] +pub struct JsonDeserializer { + #[derivative(Default(value = "default_lossy()"))] + lossy: bool, +} impl JsonDeserializer { /// Creates a new `JsonDeserializer`. - pub fn new() -> Self { - Default::default() + pub fn new(lossy: bool) -> Self { + Self { lossy } } } @@ -84,8 +119,11 @@ impl Deserializer for JsonDeserializer { return Ok(smallvec![]); } - let json: serde_json::Value = serde_json::from_slice(&bytes) - .map_err(|error| format!("Error parsing JSON: {:?}", error))?; + let json: serde_json::Value = match self.lossy { + true => serde_json::from_str(&String::from_utf8_lossy(&bytes)), + false => serde_json::from_slice(&bytes), + } + .map_err(|error| format!("Error parsing JSON: {:?}", error))?; // If the root is an Array, split it into multiple events let mut events = match json { @@ -119,8 +157,10 @@ impl Deserializer for JsonDeserializer { } impl From<&JsonDeserializerConfig> for JsonDeserializer { - fn from(_: &JsonDeserializerConfig) -> Self { - Self + fn from(config: &JsonDeserializerConfig) -> Self { + Self { + lossy: config.json.lossy, + } } } @@ -133,7 +173,7 @@ mod tests { #[test] fn deserialize_json() { let input = Bytes::from(r#"{ "foo": 123 }"#); - let deserializer = JsonDeserializer::new(); + let deserializer = JsonDeserializer::default(); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { let events = deserializer.parse(input.clone(), namespace).unwrap(); @@ -160,7 +200,7 @@ mod tests { #[test] fn deserialize_json_array() { let input = Bytes::from(r#"[{ "foo": 123 }, { "bar": 456 }]"#); - let deserializer = JsonDeserializer::new(); + let deserializer = JsonDeserializer::default(); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { let events = deserializer.parse(input.clone(), namespace).unwrap(); let mut events = events.into_iter(); @@ -197,7 +237,7 @@ mod tests { #[test] fn deserialize_skip_empty() { let input = Bytes::from(""); - let deserializer = JsonDeserializer::new(); + let deserializer = JsonDeserializer::default(); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { let events = deserializer.parse(input.clone(), namespace).unwrap(); @@ -208,7 +248,44 @@ mod tests { #[test] fn deserialize_error_invalid_json() { let input = Bytes::from("{ foo"); - let deserializer = JsonDeserializer::new(); + let deserializer = JsonDeserializer::default(); + + for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { + assert!(deserializer.parse(input.clone(), namespace).is_err()); + } + } + + #[test] + fn deserialize_lossy_replace_invalid_utf8() { + let input = Bytes::from(b"{ \"foo\": \"Hello \xF0\x90\x80World\" }".as_slice()); + let deserializer = JsonDeserializer::new(true); + + for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { + let events = deserializer.parse(input.clone(), namespace).unwrap(); + let mut events = events.into_iter(); + + { + let event = events.next().unwrap(); + let log = event.as_log(); + assert_eq!(log["foo"], b"Hello \xEF\xBF\xBDWorld".into()); + assert_eq!( + log.get(( + lookup::PathPrefix::Event, + log_schema().timestamp_key().unwrap() + )) + .is_some(), + namespace == LogNamespace::Legacy + ); + } + + assert_eq!(events.next(), None); + } + } + + #[test] + fn deserialize_non_lossy_error_invalid_utf8() { + let input = Bytes::from(b"{ \"foo\": \"Hello \xF0\x90\x80World\" }".as_slice()); + let deserializer = JsonDeserializer::new(false); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { assert!(deserializer.parse(input.clone(), namespace).is_err()); diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 8ac77f0a5fe06..d24faefb85382 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -14,7 +14,7 @@ mod syslog; use ::bytes::Bytes; use dyn_clone::DynClone; pub use gelf::{GelfDeserializer, GelfDeserializerConfig}; -pub use json::{JsonDeserializer, JsonDeserializerConfig}; +pub use json::{JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions}; pub use native::{NativeDeserializer, NativeDeserializerConfig}; pub use native_json::{NativeJsonDeserializer, NativeJsonDeserializerConfig}; use smallvec::SmallVec; diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index 6501c5e3ceb6d..1fbd05a1ef2e2 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -9,8 +9,9 @@ use bytes::{Bytes, BytesMut}; pub use error::StreamDecodingError; pub use format::{ BoxedDeserializer, BytesDeserializer, BytesDeserializerConfig, GelfDeserializer, - GelfDeserializerConfig, JsonDeserializer, JsonDeserializerConfig, NativeDeserializer, - NativeDeserializerConfig, NativeJsonDeserializer, NativeJsonDeserializerConfig, + GelfDeserializerConfig, JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions, + NativeDeserializer, NativeDeserializerConfig, NativeJsonDeserializer, + NativeJsonDeserializerConfig, }; #[cfg(feature = "syslog")] pub use format::{SyslogDeserializer, SyslogDeserializerConfig}; @@ -243,7 +244,14 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [JSON][json]. /// /// [json]: https://www.json.org/ - Json, + Json { + /// Options for the JSON deserializer. + #[serde( + default, + skip_serializing_if = "vector_core::serde::skip_serializing_if_default" + )] + json: JsonDeserializerOptions, + }, #[cfg(feature = "syslog")] /// Decodes the raw bytes as a Syslog message. @@ -284,8 +292,8 @@ impl From for DeserializerConfig { } impl From for DeserializerConfig { - fn from(_: JsonDeserializerConfig) -> Self { - Self::Json + fn from(config: JsonDeserializerConfig) -> Self { + Self::Json { json: config.json } } } @@ -307,7 +315,9 @@ impl DeserializerConfig { pub fn build(&self) -> Deserializer { match self { DeserializerConfig::Bytes => Deserializer::Bytes(BytesDeserializerConfig.build()), - DeserializerConfig::Json => Deserializer::Json(JsonDeserializerConfig.build()), + DeserializerConfig::Json { json } => { + Deserializer::Json(JsonDeserializerConfig::new(json.clone()).build()) + } #[cfg(feature = "syslog")] DeserializerConfig::Syslog => { Deserializer::Syslog(SyslogDeserializerConfig::default().build()) @@ -325,7 +335,7 @@ impl DeserializerConfig { match self { DeserializerConfig::Native => FramingConfig::LengthDelimited, DeserializerConfig::Bytes - | DeserializerConfig::Json + | DeserializerConfig::Json { .. } | DeserializerConfig::Gelf | DeserializerConfig::NativeJson => FramingConfig::NewlineDelimited { newline_delimited: Default::default(), @@ -341,7 +351,9 @@ impl DeserializerConfig { pub fn output_type(&self) -> DataType { match self { DeserializerConfig::Bytes => BytesDeserializerConfig.output_type(), - DeserializerConfig::Json => JsonDeserializerConfig.output_type(), + DeserializerConfig::Json { json } => { + JsonDeserializerConfig::new(json.clone()).output_type() + } #[cfg(feature = "syslog")] DeserializerConfig::Syslog => SyslogDeserializerConfig::default().output_type(), DeserializerConfig::Native => NativeDeserializerConfig.output_type(), @@ -354,7 +366,9 @@ impl DeserializerConfig { pub fn schema_definition(&self, log_namespace: LogNamespace) -> schema::Definition { match self { DeserializerConfig::Bytes => BytesDeserializerConfig.schema_definition(log_namespace), - DeserializerConfig::Json => JsonDeserializerConfig.schema_definition(log_namespace), + DeserializerConfig::Json { json } => { + JsonDeserializerConfig::new(json.clone()).schema_definition(log_namespace) + } #[cfg(feature = "syslog")] DeserializerConfig::Syslog => { SyslogDeserializerConfig::default().schema_definition(log_namespace) @@ -371,12 +385,12 @@ impl DeserializerConfig { pub const fn content_type(&self, framer: &FramingConfig) -> &'static str { match (&self, framer) { ( - DeserializerConfig::Json | DeserializerConfig::NativeJson, + DeserializerConfig::Json { .. } | DeserializerConfig::NativeJson, FramingConfig::NewlineDelimited { .. }, ) => "application/x-ndjson", ( DeserializerConfig::Gelf - | DeserializerConfig::Json + | DeserializerConfig::Json { .. } | DeserializerConfig::NativeJson, FramingConfig::CharacterDelimited { character_delimited: @@ -388,7 +402,7 @@ impl DeserializerConfig { ) => "application/json", (DeserializerConfig::Native, _) => "application/octet-stream", ( - DeserializerConfig::Json + DeserializerConfig::Json { .. } | DeserializerConfig::NativeJson | DeserializerConfig::Bytes | DeserializerConfig::Gelf, diff --git a/src/codecs/decoding/decoder.rs b/src/codecs/decoding/decoder.rs index 6f212f450dc83..ecf19c17d715e 100644 --- a/src/codecs/decoding/decoder.rs +++ b/src/codecs/decoding/decoder.rs @@ -122,7 +122,7 @@ mod tests { let reader = StreamReader::new(stream); let decoder = Decoder::new( Framer::NewlineDelimited(NewlineDelimitedDecoder::new()), - Deserializer::Json(JsonDeserializer::new()), + Deserializer::Json(JsonDeserializer::default()), ); let mut stream = FramedRead::new(reader, decoder); diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index b6427bd7f6ab5..bbe80843753a5 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -141,7 +141,7 @@ fn deserializer_config_to_serializer(config: &DeserializerConfig) -> encoding::S // "bytes" can be a top-level field and we aren't implicitly decoding everything into the // `message` field... but it's close enough for now. DeserializerConfig::Bytes => SerializerConfig::Text(TextSerializerConfig::default()), - DeserializerConfig::Json => SerializerConfig::Json(JsonSerializerConfig::default()), + DeserializerConfig::Json { .. } => SerializerConfig::Json(JsonSerializerConfig::default()), // TODO: We need to create an Avro serializer because, certainly, for any source decoding // the data as Avro, we can't possibly send anything else without the source just // immediately barfing. @@ -184,7 +184,9 @@ fn serializer_config_to_deserializer(config: &SerializerConfig) -> decoding::Des SerializerConfig::Avro { .. } => todo!(), SerializerConfig::Csv { .. } => todo!(), SerializerConfig::Gelf => DeserializerConfig::Gelf, - SerializerConfig::Json(_) => DeserializerConfig::Json, + SerializerConfig::Json(_) => DeserializerConfig::Json { + json: Default::default(), + }, SerializerConfig::Logfmt => todo!(), SerializerConfig::Native => DeserializerConfig::Native, SerializerConfig::NativeJson => DeserializerConfig::NativeJson, diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 4508082573d6c..bd7d521b43e5e 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -1595,7 +1595,9 @@ fn test_config_outputs() { ( "json / single output", TestCase { - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, multiple_outputs: false, want: HashMap::from([( None, @@ -1620,7 +1622,9 @@ fn test_config_outputs() { ( "json / multiple output", TestCase { - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, multiple_outputs: true, want: HashMap::from([ ( diff --git a/src/sources/http_client/client.rs b/src/sources/http_client/client.rs index 6732a4975aad7..0b8ba25dace35 100644 --- a/src/sources/http_client/client.rs +++ b/src/sources/http_client/client.rs @@ -235,7 +235,9 @@ impl ValidatableComponent for HttpClientConfig { let config = Self { endpoint: uri.to_string(), interval: Duration::from_secs(1), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, ..Default::default() }; diff --git a/src/sources/http_client/integration_tests.rs b/src/sources/http_client/integration_tests.rs index ec7b9f11d71fc..d49b7f92111fd 100644 --- a/src/sources/http_client/integration_tests.rs +++ b/src/sources/http_client/integration_tests.rs @@ -96,7 +96,9 @@ async fn collected_logs_json() { endpoint: format!("{}/logs/json.json", dufs_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, @@ -173,7 +175,9 @@ async fn unauthorized_no_auth() { endpoint: format!("{}/logs/json.json", dufs_auth_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, @@ -191,7 +195,9 @@ async fn unauthorized_wrong_auth() { endpoint: format!("{}/logs/json.json", dufs_auth_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, @@ -212,7 +218,9 @@ async fn authorized() { endpoint: format!("{}/logs/json.json", dufs_auth_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, @@ -233,7 +241,9 @@ async fn tls_invalid_ca() { endpoint: format!("{}/logs/json.json", dufs_https_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, @@ -254,7 +264,9 @@ async fn tls_valid() { endpoint: format!("{}/logs/json.json", dufs_https_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, @@ -276,7 +288,9 @@ async fn shutdown() { endpoint: format!("{}/logs/json.json", dufs_address()), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, diff --git a/src/sources/http_client/tests.rs b/src/sources/http_client/tests.rs index 8676dc5678cd4..bf1e6b007511a 100644 --- a/src/sources/http_client/tests.rs +++ b/src/sources/http_client/tests.rs @@ -78,7 +78,9 @@ async fn json_decoding_newline_delimited() { endpoint: format!("http://{}/endpoint", in_addr), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: FramingConfig::NewlineDelimited { newline_delimited: NewlineDelimitedDecoderOptions::default(), }, @@ -108,7 +110,9 @@ async fn json_decoding_character_delimited() { endpoint: format!("http://{}/endpoint", in_addr), interval: INTERVAL, query: HashMap::new(), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: FramingConfig::CharacterDelimited { character_delimited: CharacterDelimitedDecoderOptions { delimiter: b',', @@ -146,7 +150,9 @@ async fn request_query_applied() { vec!["val1".to_string(), "val2".to_string()], ), ]), - decoding: DeserializerConfig::Json, + decoding: DeserializerConfig::Json { + json: Default::default(), + }, framing: default_framing_message_based(), headers: HashMap::new(), method: HttpMethod::Get, diff --git a/src/sources/http_server.rs b/src/sources/http_server.rs index c4730fe7fee5b..50f9b3fd0bd32 100644 --- a/src/sources/http_server.rs +++ b/src/sources/http_server.rs @@ -202,11 +202,11 @@ impl SimpleHttpConfig { ), Encoding::Json => ( BytesDecoderConfig::new().into(), - JsonDeserializerConfig::new().into(), + JsonDeserializerConfig::default().into(), ), Encoding::Ndjson => ( NewlineDelimitedDecoderConfig::new().into(), - JsonDeserializerConfig::new().into(), + JsonDeserializerConfig::default().into(), ), Encoding::Binary => ( BytesDecoderConfig::new().into(), @@ -256,7 +256,9 @@ impl_generate_config_from_default!(SimpleHttpConfig); impl ValidatableComponent for SimpleHttpConfig { fn validation_configuration() -> ValidationConfiguration { let config = Self { - decoding: Some(DeserializerConfig::Json), + decoding: Some(DeserializerConfig::Json { + json: Default::default(), + }), ..Default::default() }; @@ -760,7 +762,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -810,7 +812,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -853,7 +855,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -902,7 +904,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -986,7 +988,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -1027,7 +1029,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -1106,7 +1108,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -1150,7 +1152,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; @@ -1219,7 +1221,7 @@ mod tests { EventStatus::Delivered, true, None, - Some(JsonDeserializerConfig::new().into()), + Some(JsonDeserializerConfig::default().into()), ) .await; diff --git a/website/cue/reference/components/sources/base/amqp.cue b/website/cue/reference/components/sources/base/amqp.cue index bf487fc82b63c..404b600a86097 100644 --- a/website/cue/reference/components/sources/base/amqp.cue +++ b/website/cue/reference/components/sources/base/amqp.cue @@ -49,48 +49,66 @@ base: components: sources: amqp: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue index 998223c93dc03..73b2de60e2050 100644 --- a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue @@ -52,48 +52,66 @@ base: components: sources: aws_kinesis_firehose: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index 8311de4c71911..03589426d8a57 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -138,48 +138,66 @@ base: components: sources: aws_s3: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 52cd9c2e42729..b2d469c8db6a5 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -133,48 +133,66 @@ base: components: sources: aws_sqs: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/datadog_agent.cue b/website/cue/reference/components/sources/base/datadog_agent.cue index 33d7ee2302186..b356e18ac169c 100644 --- a/website/cue/reference/components/sources/base/datadog_agent.cue +++ b/website/cue/reference/components/sources/base/datadog_agent.cue @@ -34,48 +34,66 @@ base: components: sources: datadog_agent: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/demo_logs.cue b/website/cue/reference/components/sources/base/demo_logs.cue index 534afb0a437de..d75717f71c32d 100644 --- a/website/cue/reference/components/sources/base/demo_logs.cue +++ b/website/cue/reference/components/sources/base/demo_logs.cue @@ -13,48 +13,66 @@ base: components: sources: demo_logs: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/exec.cue b/website/cue/reference/components/sources/base/exec.cue index 06c7c09fadb96..a182abfc51e2b 100644 --- a/website/cue/reference/components/sources/base/exec.cue +++ b/website/cue/reference/components/sources/base/exec.cue @@ -9,48 +9,66 @@ base: components: sources: exec: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. - - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - - This codec is **[experimental][experimental]**. - - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - - This codec is **[experimental][experimental]**. - - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/file_descriptor.cue b/website/cue/reference/components/sources/base/file_descriptor.cue index 3d0be84a25207..6bbbda11ed427 100644 --- a/website/cue/reference/components/sources/base/file_descriptor.cue +++ b/website/cue/reference/components/sources/base/file_descriptor.cue @@ -4,48 +4,66 @@ base: components: sources: file_descriptor: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. - - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - - This codec is **[experimental][experimental]**. - - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - - This codec is **[experimental][experimental]**. - - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/gcp_pubsub.cue b/website/cue/reference/components/sources/base/gcp_pubsub.cue index afd10831e91a7..f5b03561cce9e 100644 --- a/website/cue/reference/components/sources/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sources/base/gcp_pubsub.cue @@ -80,48 +80,66 @@ base: components: sources: gcp_pubsub: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 3bb613492ab97..e42f80fc2c476 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -46,48 +46,66 @@ base: components: sources: heroku_logs: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/http.cue b/website/cue/reference/components/sources/base/http.cue index c48d01eb2a4a9..12a95c494068e 100644 --- a/website/cue/reference/components/sources/base/http.cue +++ b/website/cue/reference/components/sources/base/http.cue @@ -50,46 +50,64 @@ base: components: sources: http: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: true - type: string: enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: true + type: string: enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt - """ + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } } } } diff --git a/website/cue/reference/components/sources/base/http_client.cue b/website/cue/reference/components/sources/base/http_client.cue index 13577f858e80e..25209b31e2843 100644 --- a/website/cue/reference/components/sources/base/http_client.cue +++ b/website/cue/reference/components/sources/base/http_client.cue @@ -46,48 +46,66 @@ base: components: sources: http_client: configuration: { decoding: { description: "Decoder to use on the HTTP responses." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/http_server.cue b/website/cue/reference/components/sources/base/http_server.cue index 6558067f7c8f6..9edad734be947 100644 --- a/website/cue/reference/components/sources/base/http_server.cue +++ b/website/cue/reference/components/sources/base/http_server.cue @@ -50,46 +50,64 @@ base: components: sources: http_server: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: true - type: string: enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: true + type: string: enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt - """ + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character + """ + required: false + type: bool: default: true + } } } } diff --git a/website/cue/reference/components/sources/base/kafka.cue b/website/cue/reference/components/sources/base/kafka.cue index 1c03c2e568393..30fee8d24567b 100644 --- a/website/cue/reference/components/sources/base/kafka.cue +++ b/website/cue/reference/components/sources/base/kafka.cue @@ -58,48 +58,66 @@ base: components: sources: kafka: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - This codec is **[experimental][experimental]**. + This codec is **[experimental][experimental]**. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/nats.cue b/website/cue/reference/components/sources/base/nats.cue index ca42dfb333f28..5e232b607a9e8 100644 --- a/website/cue/reference/components/sources/base/nats.cue +++ b/website/cue/reference/components/sources/base/nats.cue @@ -101,48 +101,66 @@ base: components: sources: nats: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/redis.cue b/website/cue/reference/components/sources/base/redis.cue index 03cf03f45ef65..0336931a1bfcd 100644 --- a/website/cue/reference/components/sources/base/redis.cue +++ b/website/cue/reference/components/sources/base/redis.cue @@ -19,48 +19,66 @@ base: components: sources: redis: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. - This codec is **[experimental][experimental]**. + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + This codec is **[experimental][experimental]**. - This codec is **[experimental][experimental]**. + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + This codec is **[experimental][experimental]**. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. + + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. + + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/socket.cue b/website/cue/reference/components/sources/base/socket.cue index f6236c0d1c94e..1c57034f87f8f 100644 --- a/website/cue/reference/components/sources/base/socket.cue +++ b/website/cue/reference/components/sources/base/socket.cue @@ -21,48 +21,66 @@ base: components: sources: socket: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. - - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - - This codec is **[experimental][experimental]**. - - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - - This codec is **[experimental][experimental]**. - - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } } diff --git a/website/cue/reference/components/sources/base/stdin.cue b/website/cue/reference/components/sources/base/stdin.cue index 4e5d9aef77028..7682e32ebecba 100644 --- a/website/cue/reference/components/sources/base/stdin.cue +++ b/website/cue/reference/components/sources/base/stdin.cue @@ -4,48 +4,66 @@ base: components: sources: stdin: configuration: { decoding: { description: "Configures how events are decoded from raw bytes." required: false - type: object: options: codec: { - description: "The codec to use for decoding events." - required: false - type: string: { - default: "bytes" - enum: { - bytes: "Uses the raw bytes as-is." - gelf: """ - Decodes the raw bytes as a [GELF][gelf] message. - - [gelf]: https://docs.graylog.org/docs/gelf - """ - json: """ - Decodes the raw bytes as [JSON][json]. - - [json]: https://www.json.org/ - """ - native: """ - Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. - - This codec is **[experimental][experimental]**. - - [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - native_json: """ - Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. - - This codec is **[experimental][experimental]**. - - [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue - [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs - """ - syslog: """ - Decodes the raw bytes as a Syslog message. + type: object: options: { + codec: { + description: "The codec to use for decoding events." + required: false + type: string: { + default: "bytes" + enum: { + bytes: "Uses the raw bytes as-is." + gelf: """ + Decodes the raw bytes as a [GELF][gelf] message. + + [gelf]: https://docs.graylog.org/docs/gelf + """ + json: """ + Decodes the raw bytes as [JSON][json]. + + [json]: https://www.json.org/ + """ + native: """ + Decodes the raw bytes as Vector’s [native Protocol Buffers format][vector_native_protobuf]. + + This codec is **[experimental][experimental]**. + + [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + native_json: """ + Decodes the raw bytes as Vector’s [native JSON format][vector_native_json]. + + This codec is **[experimental][experimental]**. + + [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue + [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs + """ + syslog: """ + Decodes the raw bytes as a Syslog message. + + Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the + [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + + [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt + [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + """ + } + } + } + json: { + description: "Options for the JSON deserializer." + relevant_when: "codec = \"json\"" + required: false + type: object: options: lossy: { + description: """ + Determines whether or not to replace invalid UTF-8 sequences instead of returning an error. - Decodes either as the [RFC 3164][rfc3164]-style format ("old" style) or the - [RFC 5424][rfc5424]-style format ("new" style, includes structured data). + When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. - [rfc3164]: https://www.ietf.org/rfc/rfc3164.txt - [rfc5424]: https://www.ietf.org/rfc/rfc5424.txt + [U+FFFD]: https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character """ + required: false + type: bool: default: true } } }