From decaaeb9f5d4e52c483365e526b63b5f7042738d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 15:01:39 -0400 Subject: [PATCH 01/30] chore(deps): Bump warp from 0.3.5 to 0.3.6 (#18704) Bumps [warp](https://github.com/seanmonstar/warp) from 0.3.5 to 0.3.6. - [Release notes](https://github.com/seanmonstar/warp/releases) - [Changelog](https://github.com/seanmonstar/warp/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/warp/compare/v0.3.5...v0.3.6) --- updated-dependencies: - dependency-name: warp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 43 ++++++------------------------------------- Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f51276850154..74f304872d0d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8815,18 +8815,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "tokio-tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.18.0", -] - [[package]] name = "tokio-tungstenite" version = "0.20.1" @@ -8837,7 +8825,7 @@ dependencies = [ "log", "rustls 0.21.7", "tokio", - "tungstenite 0.20.1", + "tungstenite", ] [[package]] @@ -9290,25 +9278,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" -[[package]] -name = "tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes 1.5.0", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "tungstenite" version = "0.20.1" @@ -9777,7 +9746,7 @@ dependencies = [ "tokio-postgres", "tokio-stream", "tokio-test", - "tokio-tungstenite 0.20.1", + "tokio-tungstenite", "tokio-util", "toml 0.8.1", "tonic 0.10.1", @@ -9827,7 +9796,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tokio-tungstenite 0.20.1", + "tokio-tungstenite", "url", "uuid", ] @@ -10261,9 +10230,9 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba431ef570df1287f7f8b07e376491ad54f84d26ac473489427231e1718e1f69" +checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" dependencies = [ "bytes 1.5.0", "futures-channel", @@ -10283,7 +10252,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-stream", - "tokio-tungstenite 0.18.0", + "tokio-tungstenite", "tokio-util", "tower-service", "tracing 0.1.37", diff --git a/Cargo.toml b/Cargo.toml index 75555ad2cbcc3..9a5eef1701a59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -327,7 +327,7 @@ trust-dns-proto = { version = "0.23.0", default-features = false, features = ["d typetag = { version = "0.2.13", default-features = false } url = { version = "2.4.1", default-features = false, features = ["serde"] } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } -warp = { version = "0.3.5", default-features = false } +warp = { version = "0.3.6", default-features = false } zstd = { version = "0.12.4", default-features = false } arr_macro = { version = "0.2.1" } From 737f5c36545fa6fc9c71e0ca3c779e96902bf0af Mon Sep 17 00:00:00 2001 From: Glen Oakley Date: Fri, 29 Sep 2023 19:07:22 +0000 Subject: [PATCH 02/30] feat(codecs): add support for protobuf encoding (#18598) * feat(codecs): add support for protobuf encoding * rename descriptor_set_path to desc_file * fix spelling errors * describe encode_message * factor out common test code * move protobuf test data to one directory * create common protobuf module * add map encoding and test * add enum encoding and test * add timestamp encoding and test * fix spelling again * assert -> assert_eq * remove simple comments * enums are not case-sensitive * add round trip test * spelling (maybe?) * fix clippy errors --- lib/codecs/src/common/mod.rs | 3 + lib/codecs/src/common/protobuf.rs | 36 ++ lib/codecs/src/decoding/format/mod.rs | 2 +- lib/codecs/src/decoding/format/protobuf.rs | 41 +- lib/codecs/src/decoding/mod.rs | 2 +- lib/codecs/src/encoding/format/mod.rs | 2 + lib/codecs/src/encoding/format/protobuf.rs | 468 ++++++++++++++++++ lib/codecs/src/encoding/mod.rs | 34 +- lib/codecs/src/lib.rs | 1 + .../protobuf/google/protobuf/timestamp.proto | 147 ++++++ .../{decoding => }/protobuf/person_someone.pb | 0 .../protobuf/person_someone.txt | 0 .../protobuf/person_someone3.pb | Bin .../protobuf/person_someone3.txt | 0 lib/codecs/tests/data/protobuf/test.desc | Bin 0 -> 1258 bytes lib/codecs/tests/data/protobuf/test.proto | 61 +++ .../protobuf/test_protobuf.desc | Bin .../protobuf/test_protobuf.proto | 0 .../protobuf/test_protobuf3.desc | Bin .../protobuf/test_protobuf3.proto | 0 lib/codecs/tests/protobuf.rs | 67 +++ src/codecs/encoding/config.rs | 5 + src/codecs/encoding/encoder.rs | 2 +- src/components/validation/resources/mod.rs | 17 +- src/sinks/websocket/sink.rs | 4 +- .../reference/components/sinks/base/amqp.cue | 26 + .../sinks/base/aws_cloudwatch_logs.cue | 26 + .../sinks/base/aws_kinesis_firehose.cue | 26 + .../sinks/base/aws_kinesis_streams.cue | 26 + .../components/sinks/base/aws_s3.cue | 26 + .../components/sinks/base/aws_sns.cue | 26 + .../components/sinks/base/aws_sqs.cue | 26 + .../components/sinks/base/azure_blob.cue | 26 + .../components/sinks/base/console.cue | 26 + .../reference/components/sinks/base/file.cue | 26 + .../sinks/base/gcp_chronicle_unstructured.cue | 26 + .../sinks/base/gcp_cloud_storage.cue | 26 + .../components/sinks/base/gcp_pubsub.cue | 26 + .../reference/components/sinks/base/http.cue | 26 + .../components/sinks/base/humio_logs.cue | 26 + .../reference/components/sinks/base/kafka.cue | 26 + .../reference/components/sinks/base/loki.cue | 26 + .../reference/components/sinks/base/nats.cue | 26 + .../components/sinks/base/papertrail.cue | 26 + .../components/sinks/base/pulsar.cue | 26 + .../reference/components/sinks/base/redis.cue | 26 + .../components/sinks/base/socket.cue | 26 + .../components/sinks/base/splunk_hec_logs.cue | 26 + .../components/sinks/base/webhdfs.cue | 26 + .../components/sinks/base/websocket.cue | 26 + 50 files changed, 1502 insertions(+), 40 deletions(-) create mode 100644 lib/codecs/src/common/mod.rs create mode 100644 lib/codecs/src/common/protobuf.rs create mode 100644 lib/codecs/src/encoding/format/protobuf.rs create mode 100644 lib/codecs/tests/data/protobuf/google/protobuf/timestamp.proto rename lib/codecs/tests/data/{decoding => }/protobuf/person_someone.pb (100%) rename lib/codecs/tests/data/{decoding => }/protobuf/person_someone.txt (100%) rename lib/codecs/tests/data/{decoding => }/protobuf/person_someone3.pb (100%) rename lib/codecs/tests/data/{decoding => }/protobuf/person_someone3.txt (100%) create mode 100644 lib/codecs/tests/data/protobuf/test.desc create mode 100644 lib/codecs/tests/data/protobuf/test.proto rename lib/codecs/tests/data/{decoding => }/protobuf/test_protobuf.desc (100%) rename lib/codecs/tests/data/{decoding => }/protobuf/test_protobuf.proto (100%) rename lib/codecs/tests/data/{decoding => }/protobuf/test_protobuf3.desc (100%) rename lib/codecs/tests/data/{decoding => }/protobuf/test_protobuf3.proto (100%) create mode 100644 lib/codecs/tests/protobuf.rs diff --git a/lib/codecs/src/common/mod.rs b/lib/codecs/src/common/mod.rs new file mode 100644 index 0000000000000..230f3b31d2f97 --- /dev/null +++ b/lib/codecs/src/common/mod.rs @@ -0,0 +1,3 @@ +//! A collection of common utility features used by both encoding and decoding logic. + +pub mod protobuf; diff --git a/lib/codecs/src/common/protobuf.rs b/lib/codecs/src/common/protobuf.rs new file mode 100644 index 0000000000000..c50321c211a23 --- /dev/null +++ b/lib/codecs/src/common/protobuf.rs @@ -0,0 +1,36 @@ +use prost_reflect::{DescriptorPool, MessageDescriptor}; +use std::path::Path; + +/// Load a `MessageDescriptor` from a specific message type from the given descriptor set file. +/// +/// The path should point to the output of `protoc -o ...` +pub fn get_message_descriptor( + descriptor_set_path: &Path, + message_type: &str, +) -> vector_common::Result { + let b = std::fs::read(descriptor_set_path).map_err(|e| { + format!("Failed to open protobuf desc file '{descriptor_set_path:?}': {e}",) + })?; + let pool = DescriptorPool::decode(b.as_slice()).map_err(|e| { + format!("Failed to parse protobuf desc file '{descriptor_set_path:?}': {e}") + })?; + pool.get_message_by_name(message_type).ok_or_else(|| { + format!("The message type '{message_type}' could not be found in '{descriptor_set_path:?}'") + .into() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_get_message_descriptor() { + let path = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/data/protobuf/test.desc"); + let message_descriptor = get_message_descriptor(&path, "test.Integers").unwrap(); + assert_eq!("Integers", message_descriptor.name()); + assert_eq!(4, message_descriptor.fields().count()); + } +} diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 5172705448dc2..52c4557d7f57f 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -20,7 +20,7 @@ pub use native::{NativeDeserializer, NativeDeserializerConfig}; pub use native_json::{ NativeJsonDeserializer, NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, }; -pub use protobuf::{ProtobufDeserializer, ProtobufDeserializerConfig}; +pub use protobuf::{ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions}; use smallvec::SmallVec; #[cfg(feature = "syslog")] pub use syslog::{SyslogDeserializer, SyslogDeserializerConfig, SyslogDeserializerOptions}; diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 71bd98c1567fa..65be4990b0905 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -1,12 +1,11 @@ use std::collections::BTreeMap; -use std::fs; use std::path::PathBuf; use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use ordered_float::NotNan; -use prost_reflect::{DescriptorPool, DynamicMessage, MessageDescriptor, ReflectMessage}; +use prost_reflect::{DynamicMessage, MessageDescriptor, ReflectMessage}; use smallvec::{smallvec, SmallVec}; use vector_config::configurable_component; use vector_core::event::LogEvent; @@ -17,6 +16,8 @@ use vector_core::{ }; use vrl::value::Kind; +use crate::common::protobuf::get_message_descriptor; + use super::Deserializer; /// Config used to build a `ProtobufDeserializer`. @@ -73,10 +74,10 @@ impl ProtobufDeserializerConfig { #[derivative(Default)] pub struct ProtobufDeserializerOptions { /// Path to desc file - desc_file: PathBuf, + pub desc_file: PathBuf, /// message type. e.g package.message - message_type: String, + pub message_type: String, } /// Deserializer that builds `Event`s from a byte frame containing protobuf. @@ -90,19 +91,6 @@ impl ProtobufDeserializer { pub fn new(message_descriptor: MessageDescriptor) -> Self { Self { message_descriptor } } - - fn get_message_descriptor( - desc_file: &PathBuf, - message_type: String, - ) -> vector_common::Result { - let b = fs::read(desc_file) - .map_err(|e| format!("Failed to open protobuf desc file '{desc_file:?}': {e}",))?; - let pool = DescriptorPool::decode(b.as_slice()) - .map_err(|e| format!("Failed to parse protobuf desc file '{desc_file:?}': {e}"))?; - Ok(pool.get_message_by_name(&message_type).unwrap_or_else(|| { - panic!("The message type '{message_type}' could not be found in '{desc_file:?}'") - })) - } } impl Deserializer for ProtobufDeserializer { @@ -137,10 +125,8 @@ impl Deserializer for ProtobufDeserializer { impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer { type Error = vector_common::Error; fn try_from(config: &ProtobufDeserializerConfig) -> vector_common::Result { - let message_descriptor = ProtobufDeserializer::get_message_descriptor( - &config.protobuf.desc_file, - config.protobuf.message_type.clone(), - )?; + let message_descriptor = + get_message_descriptor(&config.protobuf.desc_file, &config.protobuf.message_type)?; Ok(Self::new(message_descriptor)) } } @@ -245,8 +231,7 @@ mod tests { use super::*; fn test_data_dir() -> PathBuf { - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) - .join("tests/data/decoding/protobuf") + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/data/protobuf") } fn parse_and_validate( @@ -256,11 +241,7 @@ mod tests { validate_log: fn(&LogEvent), ) { let input = Bytes::from(protobuf_bin_message); - let message_descriptor = ProtobufDeserializer::get_message_descriptor( - &protobuf_desc_path, - message_type.to_string(), - ) - .unwrap(); + let message_descriptor = get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); let deserializer = ProtobufDeserializer::new(message_descriptor); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { @@ -352,9 +333,9 @@ mod tests { #[test] fn deserialize_error_invalid_protobuf() { let input = Bytes::from("{ foo"); - let message_descriptor = ProtobufDeserializer::get_message_descriptor( + let message_descriptor = get_message_descriptor( &test_data_dir().join("test_protobuf.desc"), - "test_protobuf.Person".to_string(), + "test_protobuf.Person", ) .unwrap(); let deserializer = ProtobufDeserializer::new(message_descriptor); diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index f4465fcacb434..0b9b451a55138 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -12,7 +12,7 @@ pub use format::{ GelfDeserializerConfig, GelfDeserializerOptions, JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions, NativeDeserializer, NativeDeserializerConfig, NativeJsonDeserializer, NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, ProtobufDeserializer, - ProtobufDeserializerConfig, + ProtobufDeserializerConfig, ProtobufDeserializerOptions, }; #[cfg(feature = "syslog")] pub use format::{SyslogDeserializer, SyslogDeserializerConfig, SyslogDeserializerOptions}; diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index efff723f65b46..e61f7cae0bb96 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -11,6 +11,7 @@ mod json; mod logfmt; mod native; mod native_json; +mod protobuf; mod raw_message; mod text; @@ -24,6 +25,7 @@ pub use json::{JsonSerializer, JsonSerializerConfig}; pub use logfmt::{LogfmtSerializer, LogfmtSerializerConfig}; pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; +pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; pub use text::{TextSerializer, TextSerializerConfig}; use vector_core::event::Event; diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs new file mode 100644 index 0000000000000..a67a631edf3eb --- /dev/null +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -0,0 +1,468 @@ +use crate::common::protobuf::get_message_descriptor; +use crate::encoding::BuildError; +use bytes::BytesMut; +use chrono::Timelike; +use prost::Message; +use prost_reflect::{DynamicMessage, FieldDescriptor, Kind, MapKey, MessageDescriptor}; +use std::collections::HashMap; +use std::path::PathBuf; +use tokio_util::codec::Encoder; +use vector_core::{ + config::DataType, + event::{Event, Value}, + schema, +}; + +/// Config used to build a `ProtobufSerializer`. +#[crate::configurable_component] +#[derive(Debug, Clone)] +pub struct ProtobufSerializerConfig { + /// Options for the Protobuf serializer. + pub protobuf: ProtobufSerializerOptions, +} + +impl ProtobufSerializerConfig { + /// Build the `ProtobufSerializer` from this configuration. + pub fn build(&self) -> Result { + let message_descriptor = + get_message_descriptor(&self.protobuf.desc_file, &self.protobuf.message_type)?; + Ok(ProtobufSerializer { message_descriptor }) + } + + /// The data type of events that are accepted by `ProtobufSerializer`. + pub fn input_type(&self) -> DataType { + DataType::Log.and(DataType::Trace) + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + // While technically we support `Value` variants that can't be losslessly serialized to + // Protobuf, we don't want to enforce that limitation to users yet. + schema::Requirement::empty() + } +} + +/// Protobuf serializer options. +#[crate::configurable_component] +#[derive(Debug, Clone)] +pub struct ProtobufSerializerOptions { + /// The path to the protobuf descriptor set file. + /// + /// This file is the output of `protoc -o ...` + #[configurable(metadata(docs::examples = "/etc/vector/protobuf_descriptor_set.desc"))] + pub desc_file: PathBuf, + + /// The name of the message type to use for serializing. + #[configurable(metadata(docs::examples = "package.Message"))] + pub message_type: String, +} + +/// Serializer that converts an `Event` to bytes using the Protobuf format. +#[derive(Debug, Clone)] +pub struct ProtobufSerializer { + /// The protobuf message definition to use for serialization. + message_descriptor: MessageDescriptor, +} + +/// Convert a single raw vector `Value` into a protobuf `Value`. +/// +/// Unlike `convert_value`, this ignores any field metadata such as cardinality. +fn convert_value_raw( + value: Value, + kind: &prost_reflect::Kind, +) -> Result { + let kind_str = value.kind_str().to_owned(); + match (value, kind) { + (Value::Boolean(b), Kind::Bool) => Ok(prost_reflect::Value::Bool(b)), + (Value::Bytes(b), Kind::Bytes) => Ok(prost_reflect::Value::Bytes(b)), + (Value::Bytes(b), Kind::String) => Ok(prost_reflect::Value::String( + String::from_utf8_lossy(&b).into_owned(), + )), + (Value::Bytes(b), Kind::Enum(descriptor)) => { + let string = String::from_utf8_lossy(&b).into_owned(); + if let Some(d) = descriptor + .values() + .find(|v| v.name().eq_ignore_ascii_case(&string)) + { + Ok(prost_reflect::Value::EnumNumber(d.number())) + } else { + Err(format!( + "Enum `{}` has no value that matches string '{}'", + descriptor.full_name(), + string + ) + .into()) + } + } + (Value::Float(f), Kind::Double) => Ok(prost_reflect::Value::F64(f.into_inner())), + (Value::Float(f), Kind::Float) => Ok(prost_reflect::Value::F32(f.into_inner() as f32)), + (Value::Integer(i), Kind::Int32) => Ok(prost_reflect::Value::I32(i as i32)), + (Value::Integer(i), Kind::Int64) => Ok(prost_reflect::Value::I64(i)), + (Value::Integer(i), Kind::Sint32) => Ok(prost_reflect::Value::I32(i as i32)), + (Value::Integer(i), Kind::Sint64) => Ok(prost_reflect::Value::I64(i)), + (Value::Integer(i), Kind::Sfixed32) => Ok(prost_reflect::Value::I32(i as i32)), + (Value::Integer(i), Kind::Sfixed64) => Ok(prost_reflect::Value::I64(i)), + (Value::Integer(i), Kind::Uint32) => Ok(prost_reflect::Value::U32(i as u32)), + (Value::Integer(i), Kind::Uint64) => Ok(prost_reflect::Value::U64(i as u64)), + (Value::Integer(i), Kind::Fixed32) => Ok(prost_reflect::Value::U32(i as u32)), + (Value::Integer(i), Kind::Fixed64) => Ok(prost_reflect::Value::U64(i as u64)), + (Value::Integer(i), Kind::Enum(_)) => Ok(prost_reflect::Value::EnumNumber(i as i32)), + (Value::Object(o), Kind::Message(message_descriptor)) => { + if message_descriptor.is_map_entry() { + let value_field = message_descriptor + .get_field_by_name("value") + .ok_or("Internal error with proto map processing")?; + let mut map: HashMap = HashMap::new(); + for (key, val) in o.into_iter() { + match convert_value(&value_field, val) { + Ok(prost_val) => { + map.insert(MapKey::String(key), prost_val); + } + Err(e) => return Err(e), + } + } + Ok(prost_reflect::Value::Map(map)) + } else { + // if it's not a map, it's an actual message + Ok(prost_reflect::Value::Message(encode_message( + message_descriptor, + Value::Object(o), + )?)) + } + } + (Value::Regex(r), Kind::String) => Ok(prost_reflect::Value::String(r.as_str().to_owned())), + (Value::Regex(r), Kind::Bytes) => Ok(prost_reflect::Value::Bytes(r.as_bytes())), + (Value::Timestamp(t), Kind::Int64) => Ok(prost_reflect::Value::I64(t.timestamp_micros())), + (Value::Timestamp(t), Kind::Message(descriptor)) + if descriptor.full_name() == "google.protobuf.Timestamp" => + { + let mut message = DynamicMessage::new(descriptor.clone()); + message.try_set_field_by_name("seconds", prost_reflect::Value::I64(t.timestamp()))?; + message + .try_set_field_by_name("nanos", prost_reflect::Value::I32(t.nanosecond() as i32))?; + Ok(prost_reflect::Value::Message(message)) + } + _ => Err(format!("Cannot encode vector `{kind_str}` into protobuf `{kind:?}`",).into()), + } +} + +/// Convert a vector `Value` into a protobuf `Value`. +fn convert_value( + field_descriptor: &FieldDescriptor, + value: Value, +) -> Result { + if let Value::Array(a) = value { + if field_descriptor.cardinality() == prost_reflect::Cardinality::Repeated { + let repeated: Result, vector_common::Error> = a + .into_iter() + .map(|v| convert_value_raw(v, &field_descriptor.kind())) + .collect(); + Ok(prost_reflect::Value::List(repeated?)) + } else { + Err("Cannot encode vector array into a non-repeated protobuf field".into()) + } + } else { + convert_value_raw(value, &field_descriptor.kind()) + } +} + +/// Convert a vector object (`Value`) into a protobuf message. +/// +/// This function can only operate on `Value::Object`s, +/// since they are the only field-based vector Value +/// and protobuf messages are defined as a collection of fields and values. +fn encode_message( + message_descriptor: &MessageDescriptor, + value: Value, +) -> Result { + let mut message = DynamicMessage::new(message_descriptor.clone()); + if let Value::Object(map) = value { + for field in message_descriptor.fields() { + match map.get(field.name()) { + None | Some(Value::Null) => message.clear_field(&field), + Some(value) => { + message.try_set_field(&field, convert_value(&field, value.clone())?)? + } + } + } + Ok(message) + } else { + Err("ProtobufSerializer only supports serializing objects".into()) + } +} + +impl ProtobufSerializer { + /// Creates a new `ProtobufSerializer`. + pub fn new(message_descriptor: MessageDescriptor) -> Self { + Self { message_descriptor } + } + + /// Get a description of the message type used in serialization. + pub fn descriptor_proto(&self) -> &prost_reflect::prost_types::DescriptorProto { + self.message_descriptor.descriptor_proto() + } +} + +impl Encoder for ProtobufSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + let message = match event { + Event::Log(log) => encode_message(&self.message_descriptor, log.into_parts().0), + Event::Metric(_) => unimplemented!(), + Event::Trace(trace) => encode_message( + &self.message_descriptor, + Value::Object(trace.into_parts().0), + ), + }?; + message.encode(buffer).map_err(Into::into) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use chrono::{DateTime, NaiveDateTime, Utc}; + use ordered_float::NotNan; + use prost_reflect::MapKey; + use std::collections::{BTreeMap, HashMap}; + + macro_rules! mfield { + ($m:expr, $f:expr) => { + $m.get_field_by_name($f).unwrap().into_owned() + }; + } + + fn test_message_descriptor(message_type: &str) -> MessageDescriptor { + let path = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/data/protobuf/test.desc"); + get_message_descriptor(&path, &format!("test.{message_type}")).unwrap() + } + + #[test] + fn test_encode_integers() { + let message = encode_message( + &test_message_descriptor("Integers"), + Value::Object(BTreeMap::from([ + ("i32".into(), Value::Integer(-1234)), + ("i64".into(), Value::Integer(-9876)), + ("u32".into(), Value::Integer(1234)), + ("u64".into(), Value::Integer(9876)), + ])), + ) + .unwrap(); + assert_eq!(Some(-1234), mfield!(message, "i32").as_i32()); + assert_eq!(Some(-9876), mfield!(message, "i64").as_i64()); + assert_eq!(Some(1234), mfield!(message, "u32").as_u32()); + assert_eq!(Some(9876), mfield!(message, "u64").as_u64()); + } + + #[test] + fn test_encode_floats() { + let message = encode_message( + &test_message_descriptor("Floats"), + Value::Object(BTreeMap::from([ + ("d".into(), Value::Float(NotNan::new(11.0).unwrap())), + ("f".into(), Value::Float(NotNan::new(2.0).unwrap())), + ])), + ) + .unwrap(); + assert_eq!(Some(11.0), mfield!(message, "d").as_f64()); + assert_eq!(Some(2.0), mfield!(message, "f").as_f32()); + } + + #[test] + fn test_encode_bytes() { + let bytes = Bytes::from(vec![0, 1, 2, 3]); + let message = encode_message( + &test_message_descriptor("Bytes"), + Value::Object(BTreeMap::from([ + ("text".into(), Value::Bytes(Bytes::from("vector"))), + ("binary".into(), Value::Bytes(bytes.clone())), + ])), + ) + .unwrap(); + assert_eq!(Some("vector"), mfield!(message, "text").as_str()); + assert_eq!(Some(&bytes), mfield!(message, "binary").as_bytes()); + } + + #[test] + fn test_encode_map() { + let message = encode_message( + &test_message_descriptor("Map"), + Value::Object(BTreeMap::from([ + ( + "names".into(), + Value::Object(BTreeMap::from([ + ("forty-four".into(), Value::Integer(44)), + ("one".into(), Value::Integer(1)), + ])), + ), + ( + "people".into(), + Value::Object(BTreeMap::from([( + "mark".into(), + Value::Object(BTreeMap::from([ + ("nickname".into(), Value::Bytes(Bytes::from("jeff"))), + ("age".into(), Value::Integer(22)), + ])), + )])), + ), + ])), + ) + .unwrap(); + // the simpler string->primitive map + assert_eq!( + Some(&HashMap::from([ + ( + MapKey::String("forty-four".into()), + prost_reflect::Value::I32(44), + ), + (MapKey::String("one".into()), prost_reflect::Value::I32(1),), + ])), + mfield!(message, "names").as_map() + ); + // the not-simpler string->message map + let people = mfield!(message, "people").as_map().unwrap().to_owned(); + assert_eq!(1, people.len()); + assert_eq!( + Some("jeff"), + mfield!( + people[&MapKey::String("mark".into())].as_message().unwrap(), + "nickname" + ) + .as_str() + ); + assert_eq!( + Some(22), + mfield!( + people[&MapKey::String("mark".into())].as_message().unwrap(), + "age" + ) + .as_u32() + ); + } + + #[test] + fn test_encode_enum() { + let message = encode_message( + &test_message_descriptor("Enum"), + Value::Object(BTreeMap::from([ + ("breakfast".into(), Value::Bytes(Bytes::from("tomato"))), + ("dinner".into(), Value::Bytes(Bytes::from("OLIVE"))), + ("lunch".into(), Value::Integer(0)), + ])), + ) + .unwrap(); + assert_eq!(Some(2), mfield!(message, "breakfast").as_enum_number()); + assert_eq!(Some(0), mfield!(message, "lunch").as_enum_number()); + assert_eq!(Some(1), mfield!(message, "dinner").as_enum_number()); + } + + #[test] + fn test_encode_timestamp() { + let message = encode_message( + &test_message_descriptor("Timestamp"), + Value::Object(BTreeMap::from([( + "morning".into(), + Value::Timestamp(DateTime::from_naive_utc_and_offset( + NaiveDateTime::from_timestamp_opt(8675, 309).unwrap(), + Utc, + )), + )])), + ) + .unwrap(); + let timestamp = mfield!(message, "morning").as_message().unwrap().clone(); + assert_eq!(Some(8675), mfield!(timestamp, "seconds").as_i64()); + assert_eq!(Some(309), mfield!(timestamp, "nanos").as_i32()); + } + + #[test] + fn test_encode_repeated_primitive() { + let message = encode_message( + &test_message_descriptor("RepeatedPrimitive"), + Value::Object(BTreeMap::from([( + "numbers".into(), + Value::Array(vec![ + Value::Integer(8), + Value::Integer(6), + Value::Integer(4), + ]), + )])), + ) + .unwrap(); + let list = mfield!(message, "numbers").as_list().unwrap().to_vec(); + assert_eq!(3, list.len()); + assert_eq!(Some(8), list[0].as_i64()); + assert_eq!(Some(6), list[1].as_i64()); + assert_eq!(Some(4), list[2].as_i64()); + } + + #[test] + fn test_encode_repeated_message() { + let message = encode_message( + &test_message_descriptor("RepeatedMessage"), + Value::Object(BTreeMap::from([( + "messages".into(), + Value::Array(vec![ + Value::Object(BTreeMap::from([( + "text".into(), + Value::Bytes(Bytes::from("vector")), + )])), + Value::Object(BTreeMap::from([("index".into(), Value::Integer(4444))])), + Value::Object(BTreeMap::from([ + ("text".into(), Value::Bytes(Bytes::from("protobuf"))), + ("index".into(), Value::Integer(1)), + ])), + ]), + )])), + ) + .unwrap(); + let list = mfield!(message, "messages").as_list().unwrap().to_vec(); + assert_eq!(3, list.len()); + assert_eq!( + Some("vector"), + mfield!(list[0].as_message().unwrap(), "text").as_str() + ); + assert!(!list[0].as_message().unwrap().has_field_by_name("index")); + assert!(!list[1].as_message().unwrap().has_field_by_name("t4ext")); + assert_eq!( + Some(4444), + mfield!(list[1].as_message().unwrap(), "index").as_u32() + ); + assert_eq!( + Some("protobuf"), + mfield!(list[2].as_message().unwrap(), "text").as_str() + ); + assert_eq!( + Some(1), + mfield!(list[2].as_message().unwrap(), "index").as_u32() + ); + } + + fn run_encoding_on_decoding_test_data( + filename: &str, + message_type: &str, + ) -> Result { + let test_data_dir = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join("tests/data/protobuf"); + let descriptor_set_path = test_data_dir.join(filename); + let message_descriptor = + get_message_descriptor(&descriptor_set_path, message_type).unwrap(); + encode_message( + &message_descriptor, + Value::Object(BTreeMap::from([ + ("name".into(), Value::Bytes(Bytes::from("rope"))), + ("id".into(), Value::Integer(9271)), + ])), + ) + } + + #[test] + fn test_encode_decoding_protobuf_test_data() { + // just check for the side-effect of success + run_encoding_on_decoding_test_data("test_protobuf.desc", "test_protobuf.Person").unwrap(); + run_encoding_on_decoding_test_data("test_protobuf3.desc", "test_protobuf3.Person").unwrap(); + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index f9516411720d1..098613c877cfb 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -11,7 +11,8 @@ pub use format::{ AvroSerializer, AvroSerializerConfig, AvroSerializerOptions, CsvSerializer, CsvSerializerConfig, GelfSerializer, GelfSerializerConfig, JsonSerializer, JsonSerializerConfig, LogfmtSerializer, LogfmtSerializerConfig, NativeJsonSerializer, - NativeJsonSerializerConfig, NativeSerializer, NativeSerializerConfig, RawMessageSerializer, + NativeJsonSerializerConfig, NativeSerializer, NativeSerializerConfig, ProtobufSerializer, + ProtobufSerializerConfig, ProtobufSerializerOptions, RawMessageSerializer, RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, }; pub use framing::{ @@ -222,6 +223,11 @@ pub enum SerializerConfig { /// [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs NativeJson, + /// Encodes an event as a [Protobuf][protobuf] message. + /// + /// [protobuf]: https://protobuf.dev/ + Protobuf(ProtobufSerializerConfig), + /// No encoding. /// /// This encoding uses the `message` field of a log event. @@ -284,6 +290,12 @@ impl From for SerializerConfig { } } +impl From for SerializerConfig { + fn from(config: ProtobufSerializerConfig) -> Self { + Self::Protobuf(config) + } +} + impl From for SerializerConfig { fn from(_: RawMessageSerializerConfig) -> Self { Self::RawMessage @@ -311,6 +323,7 @@ impl SerializerConfig { SerializerConfig::NativeJson => { Ok(Serializer::NativeJson(NativeJsonSerializerConfig.build())) } + SerializerConfig::Protobuf(config) => Ok(Serializer::Protobuf(config.build()?)), SerializerConfig::RawMessage => { Ok(Serializer::RawMessage(RawMessageSerializerConfig.build())) } @@ -332,9 +345,9 @@ impl SerializerConfig { // we should do so accurately, even if practically it doesn't need to be. // // [1]: https://avro.apache.org/docs/1.11.1/specification/_print/#message-framing - SerializerConfig::Avro { .. } | SerializerConfig::Native => { - FramingConfig::LengthDelimited - } + SerializerConfig::Avro { .. } + | SerializerConfig::Native + | SerializerConfig::Protobuf(_) => FramingConfig::LengthDelimited, SerializerConfig::Csv(_) | SerializerConfig::Gelf | SerializerConfig::Json(_) @@ -357,6 +370,7 @@ impl SerializerConfig { SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.input_type(), + SerializerConfig::Protobuf(config) => config.input_type(), SerializerConfig::RawMessage => RawMessageSerializerConfig.input_type(), SerializerConfig::Text(config) => config.input_type(), } @@ -374,6 +388,7 @@ impl SerializerConfig { SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.schema_requirement(), + SerializerConfig::Protobuf(config) => config.schema_requirement(), SerializerConfig::RawMessage => RawMessageSerializerConfig.schema_requirement(), SerializerConfig::Text(config) => config.schema_requirement(), } @@ -397,6 +412,8 @@ pub enum Serializer { Native(NativeSerializer), /// Uses a `NativeJsonSerializer` for serialization. NativeJson(NativeJsonSerializer), + /// Uses a `ProtobufSerializer` for serialization. + Protobuf(ProtobufSerializer), /// Uses a `RawMessageSerializer` for serialization. RawMessage(RawMessageSerializer), /// Uses a `TextSerializer` for serialization. @@ -413,6 +430,7 @@ impl Serializer { | Serializer::Logfmt(_) | Serializer::Text(_) | Serializer::Native(_) + | Serializer::Protobuf(_) | Serializer::RawMessage(_) => false, } } @@ -433,6 +451,7 @@ impl Serializer { | Serializer::Logfmt(_) | Serializer::Text(_) | Serializer::Native(_) + | Serializer::Protobuf(_) | Serializer::RawMessage(_) => { panic!("Serializer does not support JSON") } @@ -482,6 +501,12 @@ impl From for Serializer { } } +impl From for Serializer { + fn from(serializer: ProtobufSerializer) -> Self { + Self::Protobuf(serializer) + } +} + impl From for Serializer { fn from(serializer: RawMessageSerializer) -> Self { Self::RawMessage(serializer) @@ -506,6 +531,7 @@ impl tokio_util::codec::Encoder for Serializer { Serializer::Logfmt(serializer) => serializer.encode(event, buffer), Serializer::Native(serializer) => serializer.encode(event, buffer), Serializer::NativeJson(serializer) => serializer.encode(event, buffer), + Serializer::Protobuf(serializer) => serializer.encode(event, buffer), Serializer::RawMessage(serializer) => serializer.encode(event, buffer), Serializer::Text(serializer) => serializer.encode(event, buffer), } diff --git a/lib/codecs/src/lib.rs b/lib/codecs/src/lib.rs index 0aa2c63fe1952..07adc18df67b3 100644 --- a/lib/codecs/src/lib.rs +++ b/lib/codecs/src/lib.rs @@ -4,6 +4,7 @@ #![deny(missing_docs)] #![deny(warnings)] +mod common; pub mod decoding; pub mod encoding; pub mod gelf; diff --git a/lib/codecs/tests/data/protobuf/google/protobuf/timestamp.proto b/lib/codecs/tests/data/protobuf/google/protobuf/timestamp.proto new file mode 100644 index 0000000000000..3b2df6d91168e --- /dev/null +++ b/lib/codecs/tests/data/protobuf/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone.pb b/lib/codecs/tests/data/protobuf/person_someone.pb similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/person_someone.pb rename to lib/codecs/tests/data/protobuf/person_someone.pb diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone.txt b/lib/codecs/tests/data/protobuf/person_someone.txt similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/person_someone.txt rename to lib/codecs/tests/data/protobuf/person_someone.txt diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone3.pb b/lib/codecs/tests/data/protobuf/person_someone3.pb similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/person_someone3.pb rename to lib/codecs/tests/data/protobuf/person_someone3.pb diff --git a/lib/codecs/tests/data/decoding/protobuf/person_someone3.txt b/lib/codecs/tests/data/protobuf/person_someone3.txt similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/person_someone3.txt rename to lib/codecs/tests/data/protobuf/person_someone3.txt diff --git a/lib/codecs/tests/data/protobuf/test.desc b/lib/codecs/tests/data/protobuf/test.desc new file mode 100644 index 0000000000000000000000000000000000000000..f12bfa7d889b8237c0cbb34d81ce79b716907f2a GIT binary patch literal 1258 zcmah|&2G~`5boNslT1nzmIB2GKo+2aR98x)sDe`>r9Wy*tyKhwOXb8HW5ropwo{r1 zK;kWsIC9{^Gw}}GV0UdNR#b`G%s1c6&Nn*)e-S+K!q5xoeiVmkIGRuPQ-4O2)R{$= zgm5z>WF9A&Z=gObICuxD2_1*bO?J>zw5yWB#oM4UhlR5gUDM=M*mt9OhYJ?3H?4L^Ro-vhXuo^&5@;AoN{PkL~^s1OQ}cW1h+u-586__MuaJmZb#}?InwP2 zWX?G%pTr#^Oo?z#g(DS?bUWrV(2s)9NfW#Y$d&aFL9Xy7vX(;7#5@44zu-+`47}Hy zRPyRX+V}zJBcD0(Lh?6B?wQ{dP%oSazY<@ak=2;HkBI&$V4v}#NjZ!1Gq`oMpHrlPS0MTiwnE4AqWx(b$Y(ae#Q)fDH zl5DnHMWP5>NAcWG$%=ycBf*@FKS@ouakDVq1>N--qp@6q+lYDGdQ_O#5=YUmfqo@P#hw2HW7cM3 literal 0 HcmV?d00001 diff --git a/lib/codecs/tests/data/protobuf/test.proto b/lib/codecs/tests/data/protobuf/test.proto new file mode 100644 index 0000000000000..8e3275b7e5394 --- /dev/null +++ b/lib/codecs/tests/data/protobuf/test.proto @@ -0,0 +1,61 @@ +// Remember to recompile `test.desc` when you update this file: +// protoc -I . -o test.desc test.proto google/protobuf/timestamp.proto + +syntax = "proto3"; + +package test; + +import "google/protobuf/timestamp.proto"; + +message Integers { + int32 i32 = 1; + int64 i64 = 2; + uint32 u32 = 3; + uint64 u64 = 4; +} + +message Floats { + double d = 1; + float f = 2; +} + +message Bytes { + string text = 1; + bytes binary = 2; +} + +message Map { + message Person { + string nickname = 1; + uint32 age = 2; + }; + map names = 1; + map people = 2; +} + +message Enum { + enum Fruit { + APPLE = 0; + OLIVE = 1; + TOMATO = 2; + } + Fruit breakfast = 1; + Fruit lunch = 2; + Fruit dinner = 3; +} + +message Timestamp { + google.protobuf.Timestamp morning = 1; +} + +message RepeatedPrimitive { + repeated int64 numbers = 1; +} + +message RepeatedMessage { + message EmbeddedMessage { + optional string text = 1; + optional uint32 index = 2; + } + repeated EmbeddedMessage messages = 1; +} diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf.desc b/lib/codecs/tests/data/protobuf/test_protobuf.desc similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/test_protobuf.desc rename to lib/codecs/tests/data/protobuf/test_protobuf.desc diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf.proto b/lib/codecs/tests/data/protobuf/test_protobuf.proto similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/test_protobuf.proto rename to lib/codecs/tests/data/protobuf/test_protobuf.proto diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc b/lib/codecs/tests/data/protobuf/test_protobuf3.desc similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc rename to lib/codecs/tests/data/protobuf/test_protobuf3.desc diff --git a/lib/codecs/tests/data/decoding/protobuf/test_protobuf3.proto b/lib/codecs/tests/data/protobuf/test_protobuf3.proto similarity index 100% rename from lib/codecs/tests/data/decoding/protobuf/test_protobuf3.proto rename to lib/codecs/tests/data/protobuf/test_protobuf3.proto diff --git a/lib/codecs/tests/protobuf.rs b/lib/codecs/tests/protobuf.rs new file mode 100644 index 0000000000000..92acbffd9c3a8 --- /dev/null +++ b/lib/codecs/tests/protobuf.rs @@ -0,0 +1,67 @@ +//! Tests for the behaviour of Protobuf serializer and deserializer (together). + +use bytes::{Bytes, BytesMut}; +use std::path::{Path, PathBuf}; +use tokio_util::codec::Encoder; +use vector_core::config::LogNamespace; + +use codecs::decoding::format::Deserializer; +use codecs::decoding::{ + ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions, +}; +use codecs::encoding::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; + +fn test_data_dir() -> PathBuf { + PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/data/protobuf") +} + +fn read_protobuf_bin_message(path: &Path) -> Bytes { + let message_raw = std::fs::read(path).unwrap(); + Bytes::copy_from_slice(&message_raw) +} + +/// Build the serializer and deserializer from common settings +fn build_serializer_pair( + desc_file: PathBuf, + message_type: String, +) -> (ProtobufSerializer, ProtobufDeserializer) { + let serializer = ProtobufSerializerConfig { + protobuf: ProtobufSerializerOptions { + desc_file: desc_file.clone(), + message_type: message_type.clone(), + }, + } + .build() + .unwrap(); + let deserializer = ProtobufDeserializerConfig { + protobuf: ProtobufDeserializerOptions { + desc_file, + message_type, + }, + } + .build() + .unwrap(); + (serializer, deserializer) +} + +#[test] +fn roundtrip_coding() { + let protobuf_message = read_protobuf_bin_message(&test_data_dir().join("person_someone.pb")); + let desc_file = test_data_dir().join("test_protobuf.desc"); + let message_type: String = "test_protobuf.Person".into(); + let (mut serializer, deserializer) = build_serializer_pair(desc_file, message_type); + + let events_original = deserializer + .parse(protobuf_message, LogNamespace::Vector) + .unwrap(); + assert_eq!(1, events_original.len()); + let mut new_message = BytesMut::new(); + serializer + .encode(events_original[0].clone(), &mut new_message) + .unwrap(); + let protobuf_message: Bytes = new_message.into(); + let events_encoded = deserializer + .parse(protobuf_message, LogNamespace::Vector) + .unwrap(); + assert_eq!(events_original, events_encoded); +} diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs index 0ae7b1648a026..c2e6b57106c4f 100644 --- a/src/codecs/encoding/config.rs +++ b/src/codecs/encoding/config.rs @@ -112,6 +112,11 @@ impl EncodingConfigWithFraming { // https://github.com/Graylog2/graylog2-server/issues/1240 CharacterDelimitedEncoder::new(0).into() } + (None, Serializer::Protobuf(_)) => { + // Protobuf uses length-delimited messages, see: + // https://developers.google.com/protocol-buffers/docs/techniques#streaming + LengthDelimitedEncoder::new().into() + } ( None, Serializer::Csv(_) diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index 3f9970fb961a2..f7bfe086e2085 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -113,7 +113,7 @@ impl Encoder { Serializer::Gelf(_) | Serializer::Json(_) | Serializer::NativeJson(_), Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), ) => "application/json", - (Serializer::Native(_), _) => "application/octet-stream", + (Serializer::Native(_), _) | (Serializer::Protobuf(_), _) => "application/octet-stream", ( Serializer::Avro(_) | Serializer::Csv(_) diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index 105503bf2f4ea..b66e9832c0f0f 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -142,7 +142,14 @@ fn deserializer_config_to_serializer(config: &DeserializerConfig) -> encoding::S // `message` field... but it's close enough for now. DeserializerConfig::Bytes => SerializerConfig::Text(TextSerializerConfig::default()), DeserializerConfig::Json { .. } => SerializerConfig::Json(JsonSerializerConfig::default()), - DeserializerConfig::Protobuf(_) => unimplemented!(), + DeserializerConfig::Protobuf(config) => { + SerializerConfig::Protobuf(codecs::encoding::ProtobufSerializerConfig { + protobuf: codecs::encoding::ProtobufSerializerOptions { + desc_file: config.protobuf.desc_file.clone(), + message_type: config.protobuf.message_type.clone(), + }, + }) + } // TODO: We need to create an Avro serializer because, certainly, for any source decoding // the data as Avro, we can't possibly send anything else without the source just // immediately barfing. @@ -189,6 +196,14 @@ fn serializer_config_to_deserializer( SerializerConfig::Logfmt => todo!(), SerializerConfig::Native => DeserializerConfig::Native, SerializerConfig::NativeJson => DeserializerConfig::NativeJson(Default::default()), + SerializerConfig::Protobuf(config) => { + DeserializerConfig::Protobuf(codecs::decoding::ProtobufDeserializerConfig { + protobuf: codecs::decoding::ProtobufDeserializerOptions { + desc_file: config.protobuf.desc_file.clone(), + message_type: config.protobuf.message_type.clone(), + }, + }) + } SerializerConfig::RawMessage | SerializerConfig::Text(_) => DeserializerConfig::Bytes, }; diff --git a/src/sinks/websocket/sink.rs b/src/sinks/websocket/sink.rs index 9cb00d2b53ea9..93590c6c4ba96 100644 --- a/src/sinks/websocket/sink.rs +++ b/src/sinks/websocket/sink.rs @@ -235,11 +235,11 @@ impl WebSocketSink { const fn should_encode_as_binary(&self) -> bool { use codecs::encoding::Serializer::{ - Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, RawMessage, Text, + Avro, Csv, Gelf, Json, Logfmt, Native, NativeJson, Protobuf, RawMessage, Text, }; match self.encoder.serializer() { - RawMessage(_) | Avro(_) | Native(_) => true, + RawMessage(_) | Avro(_) | Native(_) | Protobuf(_) => true, Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | Text(_) | NativeJson(_) => false, } } diff --git a/website/cue/reference/components/sinks/base/amqp.cue b/website/cue/reference/components/sinks/base/amqp.cue index 366520be4c1a2..a592e402c6f5d 100644 --- a/website/cue/reference/components/sinks/base/amqp.cue +++ b/website/cue/reference/components/sinks/base/amqp.cue @@ -102,6 +102,11 @@ base: components: sinks: amqp: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -236,6 +241,27 @@ base: components: sinks: amqp: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index cae0c301c8a7c..68e8b1614b1e4 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -273,6 +273,11 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -407,6 +412,27 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 6b59b70a3abfb..f1a647e602fd5 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -252,6 +252,11 @@ base: components: sinks: aws_kinesis_firehose: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -386,6 +391,27 @@ base: components: sinks: aws_kinesis_firehose: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index b1d181751b0dc..f723f9e8ad5b2 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -252,6 +252,11 @@ base: components: sinks: aws_kinesis_streams: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -386,6 +391,27 @@ base: components: sinks: aws_kinesis_streams: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index bc0c94eb51443..16b10bc9018bc 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -361,6 +361,11 @@ base: components: sinks: aws_s3: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -495,6 +500,27 @@ base: components: sinks: aws_s3: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 59d187f844953..1cf31e8956d8c 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -188,6 +188,11 @@ base: components: sinks: aws_sns: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -322,6 +327,27 @@ base: components: sinks: aws_sns: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 04a752fdf46a0..cb7b1dcd79a9e 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -188,6 +188,11 @@ base: components: sinks: aws_sqs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -322,6 +327,27 @@ base: components: sinks: aws_sqs: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 33c66f48f3208..a56680098c0fc 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -215,6 +215,11 @@ base: components: sinks: azure_blob: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -349,6 +354,27 @@ base: components: sinks: azure_blob: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/console.cue b/website/cue/reference/components/sinks/base/console.cue index bfa8a11acbfb2..9ebd1a606fd5b 100644 --- a/website/cue/reference/components/sinks/base/console.cue +++ b/website/cue/reference/components/sinks/base/console.cue @@ -86,6 +86,11 @@ base: components: sinks: console: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -220,6 +225,27 @@ base: components: sinks: console: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index b98956de02479..590a6fb945d22 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -106,6 +106,11 @@ base: components: sinks: file: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -240,6 +245,27 @@ base: components: sinks: file: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index 03d5071ac300f..5a049cd6f6ab2 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -155,6 +155,11 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -289,6 +294,27 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index 445fdfa244b5a..47f6bf3b03df7 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -239,6 +239,11 @@ base: components: sinks: gcp_cloud_storage: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -373,6 +378,27 @@ base: components: sinks: gcp_cloud_storage: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index fd9d75f13f584..0b36642444576 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -153,6 +153,11 @@ base: components: sinks: gcp_pubsub: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -287,6 +292,27 @@ base: components: sinks: gcp_pubsub: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index 84e11a38f9a5c..58fcd9c005c81 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -194,6 +194,11 @@ base: components: sinks: http: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -328,6 +333,27 @@ base: components: sinks: http: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index abd851e5963b9..2892e356c6784 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -147,6 +147,11 @@ base: components: sinks: humio_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -281,6 +286,27 @@ base: components: sinks: humio_logs: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index 630a2a7a5cd2c..fd95ee383b816 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -141,6 +141,11 @@ base: components: sinks: kafka: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -275,6 +280,27 @@ base: components: sinks: kafka: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 3dff6f06e9ecf..fe7a9160bc88b 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -198,6 +198,11 @@ base: components: sinks: loki: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -332,6 +337,27 @@ base: components: sinks: loki: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/nats.cue b/website/cue/reference/components/sinks/base/nats.cue index b290de01c6105..11d3dc126fa7d 100644 --- a/website/cue/reference/components/sinks/base/nats.cue +++ b/website/cue/reference/components/sinks/base/nats.cue @@ -186,6 +186,11 @@ base: components: sinks: nats: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -320,6 +325,27 @@ base: components: sinks: nats: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/papertrail.cue b/website/cue/reference/components/sinks/base/papertrail.cue index 99a2794e5cfee..6f0e9d17eb439 100644 --- a/website/cue/reference/components/sinks/base/papertrail.cue +++ b/website/cue/reference/components/sinks/base/papertrail.cue @@ -86,6 +86,11 @@ base: components: sinks: papertrail: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -220,6 +225,27 @@ base: components: sinks: papertrail: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/pulsar.cue b/website/cue/reference/components/sinks/base/pulsar.cue index ddfd315be19a5..428c4f42791e0 100644 --- a/website/cue/reference/components/sinks/base/pulsar.cue +++ b/website/cue/reference/components/sinks/base/pulsar.cue @@ -180,6 +180,11 @@ base: components: sinks: pulsar: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -314,6 +319,27 @@ base: components: sinks: pulsar: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index 211e84aa8b6ac..6d2e1964de935 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -139,6 +139,11 @@ base: components: sinks: redis: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -273,6 +278,27 @@ base: components: sinks: redis: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index 6d08b5a959d80..99c353f604166 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -98,6 +98,11 @@ base: components: sinks: socket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -232,6 +237,27 @@ base: components: sinks: socket: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index a0f031235fcc5..e79d4cdb46261 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -197,6 +197,11 @@ base: components: sinks: splunk_hec_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -331,6 +336,27 @@ base: components: sinks: splunk_hec_logs: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index 9c13ce1bb894f..0dd49a0d683b8 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -147,6 +147,11 @@ base: components: sinks: webhdfs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -281,6 +286,27 @@ base: components: sinks: webhdfs: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false diff --git a/website/cue/reference/components/sinks/base/websocket.cue b/website/cue/reference/components/sinks/base/websocket.cue index 3e1f2b6277baf..c9c30e5e170ec 100644 --- a/website/cue/reference/components/sinks/base/websocket.cue +++ b/website/cue/reference/components/sinks/base/websocket.cue @@ -133,6 +133,11 @@ base: components: sinks: websocket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + protobuf: """ + Encodes an event as a [Protobuf][protobuf] message. + + [protobuf]: https://protobuf.dev/ + """ raw_message: """ No encoding. @@ -267,6 +272,27 @@ base: components: sinks: websocket: configuration: { required: false type: array: items: type: string: {} } + protobuf: { + description: "Options for the Protobuf serializer." + relevant_when: "codec = \"protobuf\"" + required: true + type: object: options: { + desc_file: { + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -o ...` + """ + required: true + type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] + } + message_type: { + description: "The name of the message type to use for serializing." + required: true + type: string: examples: ["package.Message"] + } + } + } timestamp_format: { description: "Format used for timestamp fields." required: false From d0e605ec4192d379c796ec771d3f12c6f8bda0c9 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Fri, 29 Sep 2023 13:07:32 -0600 Subject: [PATCH 03/30] chore(deps): Add more `dependabot` groups (#18719) * chore: Add more dependabot groups * Exclude the dependabot.yml file from spell checking * Fix dependabot yaml path --- .github/actions/spelling/excludes.txt | 1 + .github/dependabot.yml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt index f7674ed58ce6c..447e5d902a851 100644 --- a/.github/actions/spelling/excludes.txt +++ b/.github/actions/spelling/excludes.txt @@ -70,6 +70,7 @@ \.zst$ ^\.github/actions/spelling/ ^\Q.cargo/config.toml\E$ +^\Q.github/dependabot.yml\E$ ^\Q.github/workflows/gardener_issue_comment.yml\E$ ^\Q.github/workflows/spelling.yml\E$ ^\Qbenches/codecs/moby_dick.txt\E$ diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 99fbdd1c6fccc..435dd1bb3ca24 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,9 +11,21 @@ updates: prefix: "chore(deps)" open-pull-requests-limit: 100 groups: + amq: + patterns: + - "amq-*" + aws: + patterns: + - "aws-*" azure: patterns: - "azure_*" + clap: + patterns: + - "clap*" + phf: + patterns: + - "phf*" prost: patterns: - "prost" @@ -22,6 +34,12 @@ updates: patterns: - "tonic" - "tonic-*" + wasm-bindgen: + patterns: + - "wasm-bindgen-*" + zstd: + patterns: + - "zstd*" - package-ecosystem: "docker" directory: "/distribution/docker/" schedule: From 96def01c488bb122acb680618bf152254b6a6ae6 Mon Sep 17 00:00:00 2001 From: Sharon Xie Date: Fri, 29 Sep 2023 12:07:36 -0700 Subject: [PATCH 04/30] docs(external docs): Fix metrics test example (#18725) Update metrics test example in unit-tests.md --- .../content/en/docs/reference/configuration/unit-tests.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/content/en/docs/reference/configuration/unit-tests.md b/website/content/en/docs/reference/configuration/unit-tests.md index 3a292058bca56..a0909e0d3f32a 100644 --- a/website/content/en/docs/reference/configuration/unit-tests.md +++ b/website/content/en/docs/reference/configuration/unit-tests.md @@ -415,7 +415,7 @@ tags.environment = env name = "add_unique_id_test" [[tests.inputs]] -insert_at = "add_unique_id_to_metric" +insert_at = "add_env_to_metric" type = "metric" [tests.inputs.metric] @@ -424,13 +424,13 @@ kind = "absolute" counter = { value = 1 } [[tests.outputs]] -extract_from = "add_unique_id_to_metric" +extract_from = "add_env_to_metric" [[tests.outputs.conditions]] type = "vrl" source = ''' assert_eq!(.name, "website_hits") -assert_eq!(.kind, "absolute) +assert_eq!(.kind, "absolute") assert_eq!(.tags.environment, "production") ''' ``` From f98cd5d3a5558b459fce6c3b0afb0babb533b10c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 19:11:33 +0000 Subject: [PATCH 05/30] chore(ci): Bump check-spelling/check-spelling from 0.0.21 to 0.0.22 (#18723) Bumps [check-spelling/check-spelling](https://github.com/check-spelling/check-spelling) from 0.0.21 to 0.0.22. - [Release notes](https://github.com/check-spelling/check-spelling/releases) - [Commits](https://github.com/check-spelling/check-spelling/compare/v0.0.21...v0.0.22) --- updated-dependencies: - dependency-name: check-spelling/check-spelling dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/spelling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 6bc9e822ba966..8c123d84a0ed8 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -86,7 +86,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@v0.0.21 + uses: check-spelling/check-spelling@v0.0.22 with: suppress_push_for_open_pull_request: 1 checkout: true From ff745abb2a0c46f05f5aa894e42b679f45247087 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 29 Sep 2023 13:07:13 -0700 Subject: [PATCH 06/30] chore(releasing): Add known issue for 0.33.0 debian packaging regression (#18727) Regression: #18718 Signed-off-by: Jesse Szwedko --- website/cue/reference/releases/0.33.0.cue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/website/cue/reference/releases/0.33.0.cue b/website/cue/reference/releases/0.33.0.cue index 0b6f4ba84c9d9..900186f10278d 100644 --- a/website/cue/reference/releases/0.33.0.cue +++ b/website/cue/reference/releases/0.33.0.cue @@ -25,7 +25,9 @@ releases: "0.33.0": { """ - known_issues: [] + known_issues: [ + "The Debian package installer will overwrite existing `/etc/default/vector` and `/etc/vector/vector.yaml` files. This will be fixed in v0.33.1.", + ] changelog: [ { From 93d7af46f465693a315b0ee69df5f91329742926 Mon Sep 17 00:00:00 2001 From: Doug Smith Date: Fri, 29 Sep 2023 17:13:07 -0400 Subject: [PATCH 07/30] fix(docker source): do not emit component error for out of order logs (#18649) * fix(docker source): do not emit component error for out of order logs * fmt --- src/internal_events/docker_logs.rs | 25 -------------- src/sources/docker_logs/mod.rs | 52 ++++++++++++++++++------------ 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/src/internal_events/docker_logs.rs b/src/internal_events/docker_logs.rs index 0447624b01630..6992eb1b7ead4 100644 --- a/src/internal_events/docker_logs.rs +++ b/src/internal_events/docker_logs.rs @@ -186,28 +186,3 @@ impl InternalEvent for DockerLogsLoggingDriverUnsupportedError<'_> { counter!("logging_driver_errors_total", 1); } } - -#[derive(Debug)] -pub struct DockerLogsReceivedOutOfOrderError<'a> { - pub timestamp_str: &'a str, - pub container_id: &'a str, -} - -impl InternalEvent for DockerLogsReceivedOutOfOrderError<'_> { - fn emit(self) { - error!( - message = "Received out of order log message.", - error_type = error_type::CONDITION_FAILED, - stage = error_stage::RECEIVING, - container_id = ?self.container_id, - timestamp = ?self.timestamp_str, - internal_log_rate_limit = true, - ); - counter!( - "component_errors_total", 1, - "error_type" => error_type::CONDITION_FAILED, - "stage" => error_stage::RECEIVING, - "container_id" => self.container_id.to_owned(), - ); - } -} diff --git a/src/sources/docker_logs/mod.rs b/src/sources/docker_logs/mod.rs index c597c5ea5574f..ba18aa07d012b 100644 --- a/src/sources/docker_logs/mod.rs +++ b/src/sources/docker_logs/mod.rs @@ -37,8 +37,7 @@ use crate::{ DockerLogsCommunicationError, DockerLogsContainerEventReceived, DockerLogsContainerMetadataFetchError, DockerLogsContainerUnwatch, DockerLogsContainerWatch, DockerLogsEventsReceived, - DockerLogsLoggingDriverUnsupportedError, DockerLogsReceivedOutOfOrderError, - DockerLogsTimestampParseError, StreamClosedError, + DockerLogsLoggingDriverUnsupportedError, DockerLogsTimestampParseError, StreamClosedError, }, line_agg::{self, LineAgg}, shutdown::ShutdownSignal, @@ -793,7 +792,7 @@ impl EventStreamBuilder { error = v.as_ref().err().cloned(); ready(v.is_ok()) }) - .filter_map(|v| ready(v.unwrap())) + .filter_map(|v| ready(v.ok().flatten())) .take_until(self.shutdown.clone()); let events_stream: Box + Unpin + Send> = @@ -994,25 +993,38 @@ impl ContainerLogInfo { let timestamp_str = splitter.next()?; let timestamp = match DateTime::parse_from_rfc3339(timestamp_str) { Ok(timestamp) => { - // Timestamp check + // Timestamp check. This is included to avoid processing the same log multiple times, which can + // occur when a container changes generations, and to avoid processing logs with timestamps before + // the created timestamp. match self.last_log.as_ref() { - // Received log has not already been processed. - Some(&(ref last, gen)) - if *last < timestamp || (*last == timestamp && gen == self.generation) => - { - // noop - } - // Received log is after the time the container was created. - None if self.created <= timestamp.with_timezone(&Utc) => { - // noop + Some(&(last, gen)) => { + if last < timestamp || (last == timestamp && gen == self.generation) { + // Noop - log received in order. + } else { + // Docker returns logs in order. + // If we reach this state, this log is from a previous generation of the container. + // It was already processed, so we can safely skip it. + trace!( + message = "Received log from previous container generation.", + log_timestamp = %timestamp_str, + last_log_timestamp = %last, + ); + return None; + } } - // Received log is older than the previously received entry. - _ => { - emit!(DockerLogsReceivedOutOfOrderError { - container_id: self.id.as_str(), - timestamp_str, - }); - return None; + None => { + if self.created < timestamp.with_timezone(&Utc) { + // Noop - first log to process. + } else { + // Received a log with a timestamp before that provided to the Docker API. + // This should not happen, but if it does, we can just ignore these logs. + trace!( + message = "Received log from before created timestamp.", + log_timestamp = %timestamp_str, + created_timestamp = %self.created + ); + return None; + } } } From 17bd2b1cb60861a9854a419636a74a08ab75bce2 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 29 Sep 2023 14:31:42 -0700 Subject: [PATCH 08/30] chore(ci): Add a summary if the regression workflow is skipped (#18724) * chore(ci): Add a summary if the regression workflow is skipped Due to not running automatically on PRs. Signed-off-by: Jesse Szwedko * Escape backticks Signed-off-by: Jesse Szwedko --------- Signed-off-by: Jesse Szwedko --- .github/workflows/regression.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index daf860f65507a..089e9c9097914 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -816,6 +816,13 @@ jobs: context: Regression Detection Suite status: 'success' + - name: (PR) Indicate why skipped + if: github.event_name == 'pull_request' + run: | + echo "### Workflow skipped" >> $GITHUB_STEP_SUMMARY + echo "This workflow doesn't run on PR's automatically." >> $GITHUB_STEP_SUMMARY + echo "To trigger a run, leave a comment with \`/ci-run-regression\`" >> $GITHUB_STEP_SUMMARY + - name: exit run: | echo "failed=${{ env.FAILED }}" From e44572170222dd07803998e1760fce5d7ae4e7f1 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 29 Sep 2023 15:16:11 -0700 Subject: [PATCH 09/30] fix(debian platform): Re-add `conf-files` directive for `cargo-deb` (#18726) * Revert "fix(debian platform): Remove `conf-files` directive for `cargo-deb` (#18455)" This reverts commit 40ef7c4d1f6505353aa7a17b289023243118f7e9. * Update Cargo.toml --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 9a5eef1701a59..8c76a9434073e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ debug = true name = "vector" section = "admin" maintainer-scripts = "distribution/debian/scripts/" +conf-files = ["/etc/vector/vector.yaml", "/etc/default/vector"] assets = [ ["target/release/vector", "/usr/bin/", "755"], ["config/vector.yaml", "/etc/vector/vector.yaml", "644"], From 052ed98ad08813f4fdc41c2c86362bb6e5bc86d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:52:46 +0000 Subject: [PATCH 10/30] chore(deps): Bump chrono from 0.4.30 to 0.4.31 (#18583) * chore(deps): Bump chrono from 0.4.30 to 0.4.31 Bumps [chrono](https://github.com/chronotope/chrono) from 0.4.30 to 0.4.31. - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/main/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.30...v0.4.31) --- updated-dependencies: - dependency-name: chrono dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Expand deprecated function `DateTime::timestamp_nanos` This function internally just calls `.timestamp_nanos_opt().expect()` * Change message on expect --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bruce Guenter --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/enrichment/Cargo.toml | 2 +- lib/fakedata/Cargo.toml | 2 +- lib/loki-logproto/src/lib.rs | 10 ++++++++-- lib/opentelemetry-proto/Cargo.toml | 2 +- lib/vector-api-client/Cargo.toml | 2 +- lib/vector-config/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- .../datadog/traces/apm_stats/aggregation.rs | 18 ++++++++++++++---- .../traces/apm_stats/integration_tests.rs | 4 +++- src/sinks/datadog/traces/request_builder.rs | 4 +++- src/sinks/influxdb/logs.rs | 5 ++++- src/sinks/influxdb/metrics.rs | 7 ++++++- src/sinks/influxdb/mod.rs | 8 +++++--- src/sinks/loki/integration_tests.rs | 14 ++++++++++++-- src/sinks/loki/sink.rs | 6 ++++-- src/sources/datadog_agent/integration_tests.rs | 4 +++- src/sources/splunk_hec/mod.rs | 9 ++++++--- vdev/Cargo.toml | 2 +- 20 files changed, 78 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74f304872d0d7..13c1c2492aafd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1959,9 +1959,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", diff --git a/Cargo.toml b/Cargo.toml index 8c76a9434073e..4ca02e79be0af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -253,7 +253,7 @@ bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.15.0", default-features = false, features = ["ssl", "chrono"], optional = true } bytes = { version = "1.5.0", default-features = false, features = ["serde"] } bytesize = { version = "1.3.0", default-features = false } -chrono = { version = "0.4.30", default-features = false, features = ["serde"] } +chrono = { version = "0.4.31", default-features = false, features = ["serde"] } cidr-utils = { version = "0.5.11", default-features = false } clap = { version = "4.4.3", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } colored = { version = "2.0.4", default-features = false } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 901bf6d0b45fb..45de147609dbf 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -7,6 +7,6 @@ publish = false [dependencies] arc-swap = { version = "1.6.0", default-features = false } -chrono = { version = "0.4.19", default-features = false } +chrono = { version = "0.4.31", default-features = false } dyn-clone = { version = "1.0.14", default-features = false } vrl.workspace = true diff --git a/lib/fakedata/Cargo.toml b/lib/fakedata/Cargo.toml index d4c27a51a8a7e..6dedcb99df25e 100644 --- a/lib/fakedata/Cargo.toml +++ b/lib/fakedata/Cargo.toml @@ -7,6 +7,6 @@ publish = false license = "MPL-2.0" [dependencies] -chrono = "0.4.19" +chrono = "0.4.31" fakedata_generator = "0.2.4" rand = "0.8.5" diff --git a/lib/loki-logproto/src/lib.rs b/lib/loki-logproto/src/lib.rs index 44ab526d8f8ea..ec7fee42c7613 100644 --- a/lib/loki-logproto/src/lib.rs +++ b/lib/loki-logproto/src/lib.rs @@ -103,12 +103,18 @@ mod tests { .timestamp_opt(1640244790, 0) .single() .expect("invalid timestamp"); - let entry1 = Entry(ts1.timestamp_nanos(), "hello".into()); + let entry1 = Entry( + ts1.timestamp_nanos_opt().expect("Timestamp out of range"), + "hello".into(), + ); let ts2 = Utc .timestamp_opt(1640244791, 0) .single() .expect("invalid timestamp"); - let entry2 = Entry(ts2.timestamp_nanos(), "world".into()); + let entry2 = Entry( + ts2.timestamp_nanos_opt().expect("Timestamp out of range"), + "world".into(), + ); let labels = vec![("source".into(), "protobuf-test".into())] .into_iter() .collect(); diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 66a0f377bf6e3..2dc2f93d207cb 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -11,7 +11,7 @@ tonic-build = { version = "0.10", default-features = false, features = ["prost", [dependencies] bytes = { version = "1.5.0", default-features = false, features = ["serde"] } -chrono = { version = "0.4.19", default-features = false, features = ["serde"] } +chrono = { version = "0.4.31", default-features = false, features = ["serde"] } hex = { version = "0.4.3", default-features = false, features = ["std"] } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } ordered-float = { version = "4.1.0", default-features = false } diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index bd1f8392eb34e..1591f6eb67250 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -29,7 +29,7 @@ reqwest = { version = "0.11.20", default-features = false, features = ["json"] } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect", "rustls"] } # External libs -chrono = { version = "0.4.6", default-features = false, features = ["serde"] } +chrono = { version = "0.4.31", default-features = false, features = ["serde"] } clap = { version = "4.4.3", default-features = false, features = ["derive"] } url = { version = "2.4.1", default-features = false } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 9aa7dac4e94cd..99b1bdbdc50ca 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -11,7 +11,7 @@ name = "integration" path = "tests/integration/lib.rs" [dependencies] -chrono = { version = "0.4.19", default-features = false } +chrono = { version = "0.4.31", default-features = false } chrono-tz = { version = "0.8.3", default-features = false } encoding_rs = { version = "0.8", default-features = false, features = ["alloc", "serde"] } indexmap = { version = "2.0", default-features = false, features = ["std"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 0746584417417..6ed305ded8802 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -11,7 +11,7 @@ async-stream = { version = "0.3.5", default-features = false } async-trait = { version = "0.1", default-features = false } bitmask-enum = { version = "2.2.2", default-features = false } bytes = { version = "1.5.0", default-features = false, features = ["serde"] } -chrono = { version = "0.4.19", default-features = false, features = ["serde"] } +chrono = { version = "0.4.31", default-features = false, features = ["serde"] } crossbeam-utils = { version = "0.8.16", default-features = false } db-key = { version = "0.0.5", default-features = false, optional = true } dyn-clone = { version = "1.0.14", default-features = false } diff --git a/src/sinks/datadog/traces/apm_stats/aggregation.rs b/src/sinks/datadog/traces/apm_stats/aggregation.rs index 2c7d3a580e6ad..7d990b6c803e4 100644 --- a/src/sinks/datadog/traces/apm_stats/aggregation.rs +++ b/src/sinks/datadog/traces/apm_stats/aggregation.rs @@ -124,7 +124,11 @@ impl Aggregator { pub fn new(default_api_key: Arc) -> Self { Self { buckets: BTreeMap::new(), - oldest_timestamp: align_timestamp(Utc::now().timestamp_nanos() as u64), + oldest_timestamp: align_timestamp( + Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range") as u64, + ), default_api_key, // We can't know the below fields until have received a trace event agent_env: None, @@ -228,8 +232,12 @@ impl Aggregator { let aggkey = AggregationKey::new_aggregation_from_span(span, payload_aggkey, synthetics); let start = match span.get("start") { - Some(Value::Timestamp(val)) => val.timestamp_nanos() as u64, - _ => Utc::now().timestamp_nanos() as u64, + Some(Value::Timestamp(val)) => { + val.timestamp_nanos_opt().expect("Timestamp out of range") as u64 + } + _ => Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range") as u64, }; let duration = match span.get("duration") { @@ -277,7 +285,9 @@ impl Aggregator { // Based on https://github.com/DataDog/datadog-agent/blob/cfa750c7412faa98e87a015f8ee670e5828bbe7f/pkg/trace/stats/concentrator.go#L38-L41 // , and https://github.com/DataDog/datadog-agent/blob/cfa750c7412faa98e87a015f8ee670e5828bbe7f/pkg/trace/stats/concentrator.go#L195-L207 - let now = Utc::now().timestamp_nanos() as u64; + let now = Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range") as u64; let flush_cutoff_time = if force { // flush all the remaining buckets (the Vector process is exiting) diff --git a/src/sinks/datadog/traces/apm_stats/integration_tests.rs b/src/sinks/datadog/traces/apm_stats/integration_tests.rs index 4ca0207e30371..50c49066d9dfa 100644 --- a/src/sinks/datadog/traces/apm_stats/integration_tests.rs +++ b/src/sinks/datadog/traces/apm_stats/integration_tests.rs @@ -417,7 +417,9 @@ async fn apm_stats_e2e_test_dd_agent_to_vector_correctness() { // the URLs of the Agent trace endpoints that traces will be sent to let urls = vec![trace_agent_only_url(), trace_agent_to_vector_url()]; - let start = Utc::now().timestamp_nanos(); + let start = Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range"); let duration = 1; let span_id = 3; diff --git a/src/sinks/datadog/traces/request_builder.rs b/src/sinks/datadog/traces/request_builder.rs index 64fa9f0d9c7ed..01ec1742a381e 100644 --- a/src/sinks/datadog/traces/request_builder.rs +++ b/src/sinks/datadog/traces/request_builder.rs @@ -382,7 +382,9 @@ impl DatadogTracesEncoder { _ => 0, }; let start = match span.get("start") { - Some(Value::Timestamp(val)) => val.timestamp_nanos(), + Some(Value::Timestamp(val)) => { + val.timestamp_nanos_opt().expect("Timestamp out of range") + } _ => 0, }; diff --git a/src/sinks/influxdb/logs.rs b/src/sinks/influxdb/logs.rs index 3f4ead23ca46b..f84b75daef9a6 100644 --- a/src/sinks/influxdb/logs.rs +++ b/src/sinks/influxdb/logs.rs @@ -904,7 +904,10 @@ mod integration_tests { onboarding_v2(&endpoint).await; let now = Utc::now(); - let measure = format!("vector-{}", now.timestamp_nanos()); + let measure = format!( + "vector-{}", + now.timestamp_nanos_opt().expect("Timestamp out of range") + ); let cx = SinkContext::default(); diff --git a/src/sinks/influxdb/metrics.rs b/src/sinks/influxdb/metrics.rs index 68dd5182cadf1..31faab4ffe644 100644 --- a/src/sinks/influxdb/metrics.rs +++ b/src/sinks/influxdb/metrics.rs @@ -1109,7 +1109,12 @@ mod integration_tests { acknowledgements: Default::default(), }; - let metric = format!("counter-{}", Utc::now().timestamp_nanos()); + let metric = format!( + "counter-{}", + Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range") + ); let mut events = Vec::new(); for i in 0..10 { let event = Event::Metric( diff --git a/src/sinks/influxdb/mod.rs b/src/sinks/influxdb/mod.rs index 4cefce45e3992..a2b322c74ddf8 100644 --- a/src/sinks/influxdb/mod.rs +++ b/src/sinks/influxdb/mod.rs @@ -342,7 +342,7 @@ fn encode_string(key: &str, output: &mut BytesMut) { pub(in crate::sinks) fn encode_timestamp(timestamp: Option>) -> i64 { if let Some(ts) = timestamp { - ts.timestamp_nanos() + ts.timestamp_nanos_opt().unwrap() } else { encode_timestamp(Some(Utc::now())) } @@ -390,7 +390,7 @@ pub mod test_util { pub(crate) const TOKEN: &str = "my-token"; pub(crate) fn next_database() -> String { - format!("testdb{}", Utc::now().timestamp_nanos()) + format!("testdb{}", Utc::now().timestamp_nanos_opt().unwrap()) } pub(crate) fn ts() -> DateTime { @@ -834,7 +834,9 @@ mod tests { #[test] fn test_encode_timestamp() { - let start = Utc::now().timestamp_nanos(); + let start = Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range"); assert_eq!(encode_timestamp(Some(ts())), 1542182950000000011); assert!(encode_timestamp(None) >= start) } diff --git a/src/sinks/loki/integration_tests.rs b/src/sinks/loki/integration_tests.rs index 68ec244e04c98..f93828b353956 100644 --- a/src/sinks/loki/integration_tests.rs +++ b/src/sinks/loki/integration_tests.rs @@ -210,7 +210,12 @@ async fn namespaced_timestamp() { // The timestamp of the event needs to be the timestamp set in the `norknork` // field since that was given the meaning of `timestamp`. - assert_eq!(timestamp.timestamp_nanos(), timestamps[i]); + assert_eq!( + timestamp + .timestamp_nanos_opt() + .expect("Timestamp out of range"), + timestamps[i] + ); } } @@ -675,7 +680,12 @@ async fn test_out_of_order_events( ) } for (i, ts) in timestamps.iter().enumerate() { - assert_eq!(get_timestamp(&expected[i]).timestamp_nanos(), *ts); + assert_eq!( + get_timestamp(&expected[i]) + .timestamp_nanos_opt() + .expect("Timestamp out of range"), + *ts + ); } } diff --git a/src/sinks/loki/sink.rs b/src/sinks/loki/sink.rs index 89ae9857ee746..9d9046788d04b 100644 --- a/src/sinks/loki/sink.rs +++ b/src/sinks/loki/sink.rs @@ -257,8 +257,10 @@ impl EventEncoder { self.remove_label_fields(&mut event); let timestamp = match event.as_log().get_timestamp() { - Some(Value::Timestamp(ts)) => ts.timestamp_nanos(), - _ => chrono::Utc::now().timestamp_nanos(), + Some(Value::Timestamp(ts)) => ts.timestamp_nanos_opt().expect("Timestamp out of range"), + _ => chrono::Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range"), }; if self.remove_timestamp { diff --git a/src/sources/datadog_agent/integration_tests.rs b/src/sources/datadog_agent/integration_tests.rs index 45794732d9a00..b5b7e9120b9bf 100644 --- a/src/sources/datadog_agent/integration_tests.rs +++ b/src/sources/datadog_agent/integration_tests.rs @@ -185,6 +185,8 @@ fn get_simple_trace() -> String { ] ] "#}, - Utc::now().timestamp_nanos() + Utc::now() + .timestamp_nanos_opt() + .expect("Timestamp out of range") ) } diff --git a/src/sources/splunk_hec/mod.rs b/src/sources/splunk_hec/mod.rs index 41a15547c9c6f..ac597caf2be06 100644 --- a/src/sources/splunk_hec/mod.rs +++ b/src/sources/splunk_hec/mod.rs @@ -2123,7 +2123,7 @@ mod tests { for case in cases { let sec = case.timestamp(); let millis = case.timestamp_millis(); - let nano = case.timestamp_nanos(); + let nano = case.timestamp_nanos_opt().expect("Timestamp out of range"); assert_eq!(parse_timestamp(sec).unwrap().timestamp(), case.timestamp()); assert_eq!( @@ -2131,8 +2131,11 @@ mod tests { case.timestamp_millis() ); assert_eq!( - parse_timestamp(nano).unwrap().timestamp_nanos(), - case.timestamp_nanos() + parse_timestamp(nano) + .unwrap() + .timestamp_nanos_opt() + .unwrap(), + case.timestamp_nanos_opt().expect("Timestamp out of range") ); } diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 44406dbe7fe3c..b06c06e97d6b9 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -11,7 +11,7 @@ publish = false anyhow = "1.0.75" atty = "0.2.14" cached = "0.45.1" -chrono = { version = "0.4.22", default-features = false, features = ["serde", "clock"] } +chrono = { version = "0.4.31", default-features = false, features = ["serde", "clock"] } clap = { version = "4.4.3", features = ["derive"] } clap-verbosity-flag = "2.0.1" clap_complete = "4.4.2" From 6b92a83b05872b8343df9bdb0aefd6b4ca68169b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:53:19 +0000 Subject: [PATCH 11/30] chore(deps): Bump thiserror from 1.0.48 to 1.0.49 (#18683) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.48 to 1.0.49. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.48...1.0.49) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13c1c2492aafd..3823756a684b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8546,18 +8546,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2 1.0.67", "quote 1.0.33", From 87af0bd9c175131cc63a7952fe4ea455554a310f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:53:28 +0000 Subject: [PATCH 12/30] chore(deps): Bump sha2 from 0.10.7 to 0.10.8 (#18684) Bumps [sha2](https://github.com/RustCrypto/hashes) from 0.10.7 to 0.10.8. - [Commits](https://github.com/RustCrypto/hashes/compare/sha2-v0.10.7...sha2-v0.10.8) --- updated-dependencies: - dependency-name: sha2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- vdev/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3823756a684b3..d2bffe99a1fe9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7981,9 +7981,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", diff --git a/Cargo.toml b/Cargo.toml index 4ca02e79be0af..28c5ce95fdd27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,7 +238,7 @@ ratatui = { version = "0.23.0", optional = true, default-features = false, featu # Datadog Pipelines hex = { version = "0.4.3", default-features = false, optional = true } -sha2 = { version = "0.10.7", default-features = false, optional = true } +sha2 = { version = "0.10.8", default-features = false, optional = true } # GreptimeDB greptimedb-client = { git = "https://github.com/GreptimeTeam/greptimedb-client-rust.git", rev = "bc32362adf0df17a41a95bae4221d6d8f1775656", optional = true } diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index b06c06e97d6b9..95ac24041f02a 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -35,6 +35,6 @@ reqwest = { version = "0.11", features = ["json", "blocking"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.107" serde_yaml = "0.9.25" -sha2 = "0.10.7" +sha2 = "0.10.8" tempfile = "3.8.0" toml = { version = "0.8.1", default-features = false, features = ["parse"] } From b37ce3cc9314333159f60a8e562a3c56ae32e1a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:53:38 +0000 Subject: [PATCH 13/30] chore(deps): Bump apache-avro from 0.15.0 to 0.16.0 (#18685) * chore(deps): Bump apache-avro from 0.15.0 to 0.16.0 Bumps [apache-avro](https://github.com/apache/avro) from 0.15.0 to 0.16.0. - [Release notes](https://github.com/apache/avro/releases) - [Commits](https://github.com/apache/avro/commits) --- updated-dependencies: - dependency-name: apache-avro dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * license --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Doug Smith --- Cargo.lock | 87 ++++++++++++++++++++++++------------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 5 ++- lib/codecs/Cargo.toml | 2 +- 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2bffe99a1fe9..20dfb790a607e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,11 +238,10 @@ checksum = "8f1f8f5a6f3d50d89e3797d7593a50f96bb2aaa20ca0cc7be1fb673232c91d72" [[package]] name = "apache-avro" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7" +checksum = "ceb7c683b2f8f40970b70e39ff8be514c95b96fcb9c4af87e1ed2cb2e10801a0" dependencies = [ - "byteorder", "digest", "lazy_static", "libflate", @@ -250,15 +249,14 @@ dependencies = [ "num-bigint", "quad-rand", "rand 0.8.5", - "regex", + "regex-lite", "serde", "serde_json", "strum", "strum_macros", "thiserror", - "typed-builder 0.14.0", + "typed-builder 0.16.2", "uuid", - "zerocopy", ] [[package]] @@ -2368,6 +2366,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpufeatures" version = "0.2.9" @@ -2781,6 +2788,12 @@ dependencies = [ "syn 2.0.37", ] +[[package]] +name = "dary_heap" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" + [[package]] name = "dashmap" version = "5.5.3" @@ -4864,21 +4877,25 @@ checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libflate" -version = "1.2.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05605ab2bce11bcfc0e9c635ff29ef8b2ea83f29be257ee7d730cac3ee373093" +checksum = "9f7d5654ae1795afc7ff76f4365c2c8791b0feb18e8996a96adad8ffd7c3b2bf" dependencies = [ "adler32", + "core2", "crc32fast", + "dary_heap", "libflate_lz77", ] [[package]] name = "libflate_lz77" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a734c0493409afcd49deee13c006a04e3586b9761a03543c6272c9c51f2f5a" +checksum = "be5f52fb8c451576ec6b79d3f4deb327398bc05bbdbd99021a6e77a4c855d524" dependencies = [ + "core2", + "hashbrown 0.13.1", "rle-decode-fast", ] @@ -5589,9 +5606,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -7189,6 +7206,12 @@ dependencies = [ "regex-syntax 0.7.5", ] +[[package]] +name = "regex-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f96ede7f386ba6e910092e7ccdc04176cface62abebea07ed6b46d870ed95ca2" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -8346,9 +8369,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" +checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.67", @@ -9320,13 +9343,22 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.14.0" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cba322cb9b7bc6ca048de49e83918223f35e7a86311267013afff257004870" +checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.67", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.37", ] [[package]] @@ -10729,27 +10761,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" -[[package]] -name = "zerocopy" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" -dependencies = [ - "proc-macro2 1.0.67", - "quote 1.0.33", - "syn 1.0.109", -] - [[package]] name = "zeroize" version = "1.5.7" diff --git a/Cargo.toml b/Cargo.toml index 28c5ce95fdd27..e46bc9f9504e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -246,7 +246,7 @@ greptimedb-client = { git = "https://github.com/GreptimeTeam/greptimedb-client-r # External libs arc-swap = { version = "1.6", default-features = false, optional = true } async-compression = { version = "0.4.3", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } -apache-avro = { version = "0.15.0", default-features = false, optional = true } +apache-avro = { version = "0.16.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } base64 = { version = "0.21.4", default-features = false, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 3ebe8070ad400..ba1b4e5a3b77e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -140,6 +140,7 @@ convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Couprie , Pierre Chifflier " core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core2,https://github.com/bbqsrc/core2,Apache-2.0 OR MIT,Brendan Molloy cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" crc-catalog,https://github.com/akhilles/crc-catalog,MIT OR Apache-2.0,Akhil Velagapudi @@ -160,6 +161,7 @@ curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek,BSD-3-Cl curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors cxx,https://github.com/dtolnay/cxx,MIT OR Apache-2.0,David Tolnay darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs +dary_heap,https://github.com/hanmertens/dary_heap,MIT OR Apache-2.0,Han Mertens dashmap,https://github.com/xacrimon/dashmap,MIT,Acrimon data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin data-url,https://github.com/servo/rust-url,MIT OR Apache-2.0,Simon Sapin @@ -441,6 +443,7 @@ redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez " regex-automata,https://github.com/BurntSushi/regex-automata,Unlicense OR MIT,Andrew Gallant regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-lite,https://github.com/rust-lang/regex/tree/master/regex-lite,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Project Developers regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " rend,https://github.com/djkoloski/rend,MIT,David Koloski @@ -649,8 +652,6 @@ wyz,https://github.com/myrrlyn/wyz,MIT,myrrlyn xmlparser,https://github.com/RazrFalcon/xmlparser,MIT OR Apache-2.0,Evgeniy Reizner yaml-rust,https://github.com/chyh1990/yaml-rust,MIT OR Apache-2.0,Yuheng Chen yansi,https://github.com/SergioBenitez/yansi,MIT OR Apache-2.0,Sergio Benitez -zerocopy,https://fuchsia.googlesource.com/fuchsia/+/HEAD/src/lib/zerocopy,BSD-2-Clause,Joshua Liebow-Feeser -zerocopy-derive,https://github.com/google/zerocopy,BSD-2-Clause,Joshua Liebow-Feeser zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers zstd,https://github.com/gyscos/zstd-rs,MIT,Alexandre Bury zstd-safe,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index eaa299a2b3dc0..24e2ff4280bab 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" publish = false [dependencies] -apache-avro = { version = "0.15.0", default-features = false } +apache-avro = { version = "0.16.0", default-features = false } bytes = { version = "1", default-features = false } chrono = { version = "0.4", default-features = false } csv-core = { version = "0.1.10", default-features = false } From 65643daf9f2ce3d6cc315283fd15065518de792e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:55:19 +0000 Subject: [PATCH 14/30] chore(deps): Bump indexmap from 2.0.0 to 2.0.1 (#18705) Bumps [indexmap](https://github.com/bluss/indexmap) from 2.0.0 to 2.0.1. - [Changelog](https://github.com/bluss/indexmap/blob/master/RELEASES.md) - [Commits](https://github.com/bluss/indexmap/commits) --- updated-dependencies: - dependency-name: indexmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/prometheus-parser/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20dfb790a607e..3389b348796e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,7 +500,7 @@ dependencies = [ "fnv", "futures-util", "http", - "indexmap 2.0.0", + "indexmap 2.0.1", "mime", "multer", "num-traits", @@ -590,7 +590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86046bbced96c0fab3ff5d2b3c769c0c55b0b3a7d67f9e2f2044f349f2e7d501" dependencies = [ "bytes 1.5.0", - "indexmap 2.0.0", + "indexmap 2.0.1", "serde", "serde_json", ] @@ -3372,7 +3372,7 @@ dependencies = [ "flate2", "futures 0.3.28", "glob", - "indexmap 2.0.0", + "indexmap 2.0.1", "libc", "quickcheck", "scan_fmt", @@ -4389,9 +4389,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" dependencies = [ "equivalent", "hashbrown 0.14.0", @@ -6601,7 +6601,7 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.0.1", "nom", "num_enum 0.7.0", "prost 0.12.1", @@ -7837,7 +7837,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.0.1", "itoa", "ryu", "serde", @@ -7924,7 +7924,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.0.0", + "indexmap 2.0.1", "serde", "serde_json", "serde_with_macros 3.3.0", @@ -7973,7 +7973,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.0.1", "itoa", "ryu", "serde", @@ -8902,7 +8902,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca676d9ba1a322c1b64eb8045a5ec5c0cfb0c9d08e15e9ff622589ad5221c8fe" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.0.1", "serde", "serde_spanned", "toml_datetime", @@ -9589,7 +9589,7 @@ dependencies = [ "dunce", "glob", "hex", - "indexmap 2.0.0", + "indexmap 2.0.1", "indicatif", "itertools 0.11.0", "log", @@ -9696,7 +9696,7 @@ dependencies = [ "hyper", "hyper-openssl", "hyper-proxy", - "indexmap 2.0.0", + "indexmap 2.0.1", "indoc", "infer 0.15.0", "inventory", @@ -9888,7 +9888,7 @@ dependencies = [ "crossbeam-utils", "derivative", "futures 0.3.28", - "indexmap 2.0.0", + "indexmap 2.0.1", "metrics", "nom", "ordered-float 4.1.0", @@ -9919,7 +9919,7 @@ dependencies = [ "chrono-tz", "encoding_rs", "http", - "indexmap 2.0.0", + "indexmap 2.0.1", "inventory", "no-proxy", "num-traits", @@ -9990,7 +9990,7 @@ dependencies = [ "headers", "http", "hyper-proxy", - "indexmap 2.0.0", + "indexmap 2.0.1", "metrics", "metrics-tracing-context", "metrics-util", @@ -10157,7 +10157,7 @@ dependencies = [ "hex", "hmac", "hostname", - "indexmap 2.0.0", + "indexmap 2.0.1", "indoc", "itertools 0.11.0", "lalrpop", diff --git a/Cargo.toml b/Cargo.toml index e46bc9f9504e7..8683d41fd3f3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -280,7 +280,7 @@ http-body = { version = "0.4.5", default-features = false } hyper = { version = "0.14.27", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] } hyper-openssl = { version = "0.9.2", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } -indexmap = { version = "~2.0.0", default-features = false, features = ["serde", "std"] } +indexmap = { version = "~2.0.1", default-features = false, features = ["serde", "std"] } infer = { version = "0.15.0", default-features = false, optional = true} indoc = { version = "2.0.4", default-features = false } inventory = { version = "0.3.12", default-features = false } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index d3cb1f39dd457..64fefc8e16bee 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -39,7 +39,7 @@ default-features = false features = [] [dependencies.indexmap] -version = "~2.0.0" +version = "~2.0.1" default-features = false features = ["serde"] diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index 54b0fc42dc862..8c2cad878944e 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -9,7 +9,7 @@ license = "MPL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -indexmap = "~2.0.0" +indexmap = "~2.0.1" nom = "7.1.3" num_enum = "0.7.0" prost = "0.12" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index a4c6dc4af3f68..18eb337ae2704 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,7 +48,7 @@ chrono = { version = "0.4", default-features = false, optional = true, features crossbeam-utils = { version = "0.8.16", default-features = false } derivative = { version = "2.2.0", default-features = false } futures = { version = "0.3.28", default-features = false, features = ["std"] } -indexmap = { version = "~2.0.0", default-features = false, features = ["std"] } +indexmap = { version = "~2.0.1", default-features = false, features = ["std"] } metrics = "0.21.1" nom = { version = "7", optional = true } ordered-float = { version = "4.1.0", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 6ed305ded8802..9bd5d5a64bb09 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -23,7 +23,7 @@ futures-util = { version = "0.3.28", default-features = false, features = ["std" headers = { version = "0.3.9", default-features = false } http = { version = "0.2.9", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } -indexmap = { version = "~2.0.0", default-features = false, features = ["serde", "std"] } +indexmap = { version = "~2.0.1", default-features = false, features = ["serde", "std"] } lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics = "0.21.1" metrics-tracing-context = { version = "0.14.0", default-features = false } From c95df7cf5ec4178ed2e6bdee32c6445d7e193ce4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:55:50 +0000 Subject: [PATCH 15/30] chore(deps): Bump the tonic group with 2 updates (#18714) Bumps the tonic group with 2 updates: [tonic](https://github.com/hyperium/tonic) and [tonic-build](https://github.com/hyperium/tonic). Updates `tonic` from 0.10.1 to 0.10.2 - [Changelog](https://github.com/hyperium/tonic/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/tonic/compare/v0.10.1...v0.10.2) Updates `tonic-build` from 0.10.1 to 0.10.2 - [Changelog](https://github.com/hyperium/tonic/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/tonic/compare/v0.10.1...v0.10.2) --- updated-dependencies: - dependency-name: tonic dependency-type: direct:production update-type: version-update:semver-patch dependency-group: tonic - dependency-name: tonic-build dependency-type: direct:production update-type: version-update:semver-patch dependency-group: tonic ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3389b348796e3..840bbed915a6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5960,8 +5960,8 @@ dependencies = [ "ordered-float 4.1.0", "prost 0.12.1", "prost-build 0.12.1", - "tonic 0.10.1", - "tonic-build 0.10.1", + "tonic 0.10.2", + "tonic-build 0.10.2", "vector-core", "vector-lookup", "vrl", @@ -8942,9 +8942,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c00bc15e49625f3d2f20b17082601e5e17cf27ead69e805174026c194b6664" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" dependencies = [ "async-stream", "async-trait", @@ -8987,9 +8987,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d37bb15da06ae9bb945963066baca6561b505af93a52e949a85d28558459a2" +checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" dependencies = [ "prettyplease 0.2.12", "proc-macro2 1.0.67", @@ -9781,8 +9781,8 @@ dependencies = [ "tokio-tungstenite", "tokio-util", "toml 0.8.1", - "tonic 0.10.1", - "tonic-build 0.10.1", + "tonic 0.10.2", + "tonic-build 0.10.2", "tower", "tower-http", "tower-test", @@ -10030,7 +10030,7 @@ dependencies = [ "tokio-test", "tokio-util", "toml 0.8.1", - "tonic 0.10.1", + "tonic 0.10.2", "tower", "tracing 0.1.37", "tracing-core 0.1.30", From 27b2c93523afc25113128db1561684290baa594f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:56:12 +0000 Subject: [PATCH 16/30] chore(deps): Bump clap_complete from 4.4.2 to 4.4.3 (#18716) Bumps [clap_complete](https://github.com/clap-rs/clap) from 4.4.2 to 4.4.3. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/clap_complete-v4.4.2...clap_complete-v4.4.3) --- updated-dependencies: - dependency-name: clap_complete dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- vdev/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 840bbed915a6a..d4bf3a4b5b1eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2094,9 +2094,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.4.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8baeccdb91cd69189985f87f3c7e453a3a451ab5746cf3be6acc92120bd16d24" +checksum = "e3ae8ba90b9d8b007efe66e55e48fb936272f5ca00349b5b0e89877520d35ea7" dependencies = [ "clap 4.4.5", ] diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 95ac24041f02a..2d41a85e01783 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -14,7 +14,7 @@ cached = "0.45.1" chrono = { version = "0.4.31", default-features = false, features = ["serde", "clock"] } clap = { version = "4.4.3", features = ["derive"] } clap-verbosity-flag = "2.0.1" -clap_complete = "4.4.2" +clap_complete = "4.4.3" confy = "0.5.1" directories = "5.0.1" # remove this when stabilized https://doc.rust-lang.org/stable/std/path/fn.absolute.html From 31d92c2746cd2797d8463bef64cfa035ff294481 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 23:02:58 +0000 Subject: [PATCH 17/30] chore(deps): Bump hashbrown from 0.14.0 to 0.14.1 (#18731) Bumps [hashbrown](https://github.com/rust-lang/hashbrown) from 0.14.0 to 0.14.1. - [Changelog](https://github.com/rust-lang/hashbrown/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/hashbrown/compare/v0.14.0...v0.14.1) --- updated-dependencies: - dependency-name: hashbrown dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4bf3a4b5b1eb..88768a66029f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2801,7 +2801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "lock_api", "once_cell", "parking_lot_core", @@ -3887,9 +3887,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" dependencies = [ "ahash 0.8.2", ] @@ -4394,7 +4394,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "serde", ] @@ -9685,7 +9685,7 @@ dependencies = [ "grok", "h2", "hash_hasher", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "headers", "heim", "hex", diff --git a/Cargo.toml b/Cargo.toml index 8683d41fd3f3d..d0de268665a51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -271,7 +271,7 @@ governor = { version = "0.6.0", default-features = false, features = ["dashmap", grok = { version = "2.0.0", default-features = false, optional = true } h2 = { version = "0.3.21", default-features = false, optional = true } hash_hasher = { version = "2.0.0", default-features = false } -hashbrown = { version = "0.14.0", default-features = false, optional = true, features = ["ahash"] } +hashbrown = { version = "0.14.1", default-features = false, optional = true, features = ["ahash"] } headers = { version = "0.3.9", default-features = false } hostname = { version = "0.3.1", default-features = false } http = { version = "0.2.9", default-features = false } From eda0378d4b8d65fe2b674e3f96429ffcf325aa04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 23:03:55 +0000 Subject: [PATCH 18/30] chore(deps): Bump console-subscriber from 0.1.10 to 0.2.0 (#18732) Bumps [console-subscriber](https://github.com/tokio-rs/console) from 0.1.10 to 0.2.0. - [Release notes](https://github.com/tokio-rs/console/releases) - [Commits](https://github.com/tokio-rs/console/compare/console-subscriber-v0.1.10...console-subscriber-v0.2.0) --- updated-dependencies: - dependency-name: console-subscriber dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 21 +++++++++++---------- Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88768a66029f2..58b2789149760 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2289,35 +2289,36 @@ dependencies = [ [[package]] name = "console-api" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787" dependencies = [ - "prost 0.11.9", - "prost-types 0.11.9", - "tonic 0.9.2", + "futures-core", + "prost 0.12.1", + "prost-types 0.12.1", + "tonic 0.10.1", "tracing-core 0.1.30", ] [[package]] name = "console-subscriber" -version = "0.1.10" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" +checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" dependencies = [ "console-api", "crossbeam-channel", "crossbeam-utils", - "futures 0.3.28", + "futures-task", "hdrhistogram", "humantime", - "prost-types 0.11.9", + "prost-types 0.12.1", "serde", "serde_json", "thread_local", "tokio", "tokio-stream", - "tonic 0.9.2", + "tonic 0.10.1", "tracing 0.1.37", "tracing-core 0.1.30", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index d0de268665a51..da26d897d6b98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ tokio = { version = "1.32.0", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1.14", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } -console-subscriber = { version = "0.1.10", default-features = false, optional = true } +console-subscriber = { version = "0.2.0", default-features = false, optional = true } # Tracing tracing = { version = "0.1.34", default-features = false } From 539a40f2190ec4e0fe701691b33ade93a1baca15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 23:29:44 +0000 Subject: [PATCH 19/30] chore(deps): Bump mongodb from 2.6.1 to 2.7.0 (#18703) * chore(deps): Bump mongodb from 2.6.1 to 2.7.0 Bumps [mongodb](https://github.com/mongodb/mongo-rust-driver) from 2.6.1 to 2.7.0. - [Release notes](https://github.com/mongodb/mongo-rust-driver/releases) - [Commits](https://github.com/mongodb/mongo-rust-driver/compare/v2.6.1...v2.7.0) --- updated-dependencies: - dependency-name: mongodb dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update licenses Signed-off-by: Jesse Szwedko --------- Signed-off-by: dependabot[bot] Signed-off-by: Jesse Szwedko Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jesse Szwedko --- Cargo.lock | 31 +++++++++++-------------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 - 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58b2789149760..1f5bfb12895ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1643,7 +1643,7 @@ dependencies = [ "tokio", "tokio-util", "url", - "webpki-roots 0.25.2", + "webpki-roots", "winapi", ] @@ -1706,17 +1706,17 @@ dependencies = [ [[package]] name = "bson" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aeb8bae494e49dbc330dd23cf78f6f7accee22f640ce3ab17841badaa4ce232" +checksum = "58da0ae1e701ea752cc46c1bb9f39d5ecefc7395c3ecd526261a566d4f16e0c2" dependencies = [ - "ahash 0.7.6", + "ahash 0.8.2", "base64 0.13.1", "bitvec", "hex", "indexmap 1.9.3", "js-sys", - "lazy_static", + "once_cell", "rand 0.8.5", "serde", "serde_bytes", @@ -5342,9 +5342,9 @@ checksum = "6c1a54de846c4006b88b1516731cc1f6026eb5dc4bcb186aa071ef66d40524ec" [[package]] name = "mongodb" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16928502631c0db72214720aa479c722397fe5aed6bf1c740a3830b3fe4bfcfe" +checksum = "e22d517e7e678e1c9a2983ec704b43f3b22f38b1b7a247ea3ddb36d21578bf4e" dependencies = [ "async-trait", "base64 0.13.1", @@ -5365,7 +5365,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.20.7", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_bytes", @@ -5378,13 +5378,13 @@ dependencies = [ "take_mut", "thiserror", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls 0.24.1", "tokio-util", "trust-dns-proto 0.21.2", "trust-dns-resolver", "typed-builder 0.10.0", "uuid", - "webpki-roots 0.22.5", + "webpki-roots", ] [[package]] @@ -7280,7 +7280,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.25.2", + "webpki-roots", "winreg 0.50.0", ] @@ -10419,15 +10419,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.22.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" -dependencies = [ - "webpki", -] - [[package]] name = "webpki-roots" version = "0.25.2" diff --git a/Cargo.toml b/Cargo.toml index da26d897d6b98..feae5782770f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -291,7 +291,7 @@ logfmt = { version = "0.0.2", default-features = false, optional = true } lru = { version = "0.11.1", default-features = false, optional = true } maxminddb = { version = "0.23.0", default-features = false, optional = true } md-5 = { version = "0.10", default-features = false, optional = true } -mongodb = { version = "2.6.1", default-features = false, features = ["tokio-runtime"], optional = true } +mongodb = { version = "2.7.0", default-features = false, features = ["tokio-runtime"], optional = true } async-nats = { version = "0.32.0", default-features = false, optional = true } nkeys = { version = "0.3.1", default-features = false, optional = true } nom = { version = "7.1.3", default-features = false, optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ba1b4e5a3b77e..8aaddcf107047 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -627,7 +627,6 @@ wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Ma web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm webpki,https://github.com/briansmith/webpki,ISC,Brian Smith -webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,Joseph Birr-Pixton webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,The webpki-roots Authors wepoll-ffi,https://github.com/aclysma/wepoll-ffi,MIT OR Apache-2.0 OR BSD-2-Clause,Philip Degarmo whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors From 7dce29248f96e32737d8982ddbef74ecc068ac99 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 2 Oct 2023 13:04:04 -0400 Subject: [PATCH 20/30] chore(ci): Revet bump check-spelling/check-spelling from 0.0.21 to 0.0.22 (#18742) Revert "chore(ci): Bump check-spelling/check-spelling from 0.0.21 to 0.0.22 (#18723)" This reverts commit f98cd5d3a5558b459fce6c3b0afb0babb533b10c. Co-authored-by: Jesse Szwedko --- .github/workflows/spelling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 8c123d84a0ed8..6bc9e822ba966 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -86,7 +86,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@v0.0.22 + uses: check-spelling/check-spelling@v0.0.21 with: suppress_push_for_open_pull_request: 1 checkout: true From 43428d3556d7ddf2ce76aab536087907cd5eb8c0 Mon Sep 17 00:00:00 2001 From: Nick Sollecito Date: Mon, 2 Oct 2023 16:37:07 -0400 Subject: [PATCH 21/30] [WEB-3464] Adds TrustArc cookie consent banner (#18741) * Add TrustArc cookie consent banner * Update config.toml * Update cookie-banner.js --- website/assets/js/app.js | 1 + website/assets/js/cookie-banner.js | 41 ++++++++++++++++++++++++++++++ website/config.toml | 6 +++++ website/layouts/partials/meta.html | 4 +++ 4 files changed, 52 insertions(+) create mode 100644 website/assets/js/cookie-banner.js diff --git a/website/assets/js/app.js b/website/assets/js/app.js index d4bb1c3f834cf..953c5752d7907 100644 --- a/website/assets/js/app.js +++ b/website/assets/js/app.js @@ -3,6 +3,7 @@ {{ $siteGeneration := site.Params.site_generation }} import '@ryangjchandler/spruce'; import 'alpinejs'; +import './cookie-banner' const sayHello = () => { console.log('Welcome to the Vector website and documentation!'); diff --git a/website/assets/js/cookie-banner.js b/website/assets/js/cookie-banner.js new file mode 100644 index 0000000000000..ee3044b545802 --- /dev/null +++ b/website/assets/js/cookie-banner.js @@ -0,0 +1,41 @@ +window.addEventListener('load', function () { + const { env } = document.documentElement.dataset; + + if (env !== 'development') { + // add trustarc script to head + const trustarc = document.createElement('script'); + trustarc.setAttribute('src','https://consent.trustarc.com/v2/notice/ufocto'); + document.head.appendChild(trustarc); + + // add divs + const divA = document.createElement("div"); + const divB = document.createElement("div"); + divA.id = "teconsent"; + divA.style = "cursor: pointer; color:#fff" + divB.id = "consent-banner"; + divB.style = "position:fixed; bottom:0px; right:0px; width:100%;"; + document.body.appendChild(divA); + document.body.appendChild(divB); + + // update Cookie link + this.setTimeout(function () { + const banner = document.getElementById('consent-banner'); + const prefsElement = document.getElementById('teconsent'); + const cookieLink = document.querySelector('footer a[href*="/cookies"]'); + prefsElement.className = cookieLink.className; + + if (banner) { + // listen for click and remove banner to avoid interfering with + document.addEventListener('click', function (event) { + const targetElement = event.target; + if (targetElement.matches('#truste-consent-required') || targetElement.matches('#truste-consent-button')) { + banner.remove(); + } + }); + } + + // replace Cookie link with Prefs div + return (cookieLink && document.getElementById('teconsent').innerHTML.length > 0) ? cookieLink.replaceWith(prefsElement) : false; + }, 200); + } +}); \ No newline at end of file diff --git a/website/config.toml b/website/config.toml index 1308eff1af5af..a51fd243a30dc 100644 --- a/website/config.toml +++ b/website/config.toml @@ -175,6 +175,12 @@ url = "https://www.datadoghq.com/legal/privacy/" parent = "about" weight = 3 +[[languages.en.menus.footer]] +name = "Cookies" +url = "https://www.datadoghq.com/legal/cookies/" +parent = "about" +weight = 4 + [[languages.en.menus.footer]] name = "Components" url = "/components" diff --git a/website/layouts/partials/meta.html b/website/layouts/partials/meta.html index 242f6401c9bed..2a6cd8553f131 100644 --- a/website/layouts/partials/meta.html +++ b/website/layouts/partials/meta.html @@ -16,6 +16,10 @@ {{ end }} +{{/* TrustArc */}} + + + {{ hugo.Generator }} From 7a55e5490a3a11bfdc7194380653b385b2bddbde Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Mon, 2 Oct 2023 16:52:13 -0700 Subject: [PATCH 22/30] chore(ci): Fix cookie banner style issues (#18745) * chore(ci): Fix cookie banner style issues Signed-off-by: Jesse Szwedko * spelling Signed-off-by: Jesse Szwedko --------- Signed-off-by: Jesse Szwedko --- .github/actions/spelling/excludes.txt | 6 ++++-- .github/actions/spelling/expect.txt | 5 ++++- website/assets/js/cookie-banner.js | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt index 447e5d902a851..18102653372a8 100644 --- a/.github/actions/spelling/excludes.txt +++ b/.github/actions/spelling/excludes.txt @@ -2,13 +2,13 @@ (?:^|/)(?i)COPYRIGHT (?:^|/)(?i)LICEN[CS]E (?:^|/)3rdparty/ +(?:^|/)amplify\.yml$ (?:^|/)go\.sum$ (?:^|/)package(?:-lock|)\.json$ (?:^|/)Pipfile$ (?:^|/)pyproject.toml (?:^|/)requirements(?:-dev|-doc|-test|)\.txt$ (?:^|/)vendor/ -(?:^|/)amplify\.yml$ \.a$ \.ai$ \.all-contributorsrc$ @@ -78,6 +78,9 @@ ^\Qbenches/transform/route.rs\E$ ^\Qlib/codecs/tests/data/decoding/protobuf/test_protobuf.desc\E$ ^\Qlib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc\E$ +^\Qlib/codecs/tests/data/protobuf/test.desc\E$ +^\Qlib/codecs/tests/data/protobuf/test_protobuf.desc\E$ +^\Qlib/codecs/tests/data/protobuf/test_protobuf3.desc\E$ ^\Qlib/dnsmsg-parser/benches/benches.rs\E$ ^\Qlib/dnsmsg-parser/src/dns_message_parser.rs\E$ ^\Qlib/lookup/tests/fixtures/lookup/quoted\E$ @@ -105,4 +108,3 @@ ^\Qwebsite/layouts/shortcodes/config/unit-tests.html\E$ ^lib/codecs/tests/data/native_encoding/ ignore$ - diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 628e7dc806a5d..330e4493954f9 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -840,6 +840,7 @@ posttrunc prebuild precpu preds +prefs preinst preread prereqs @@ -1074,6 +1075,7 @@ Takeaways targetgroup tarpit tcmalloc +teconsent telecom templatable templateable @@ -1133,6 +1135,8 @@ tripwires Trivago trivy Troutwine +trustarc +truste TRUSTSTORE TSDB Tsvg @@ -1224,7 +1228,6 @@ wktpointer wmem woooooow woothee -wor wordlist workdir workstreams diff --git a/website/assets/js/cookie-banner.js b/website/assets/js/cookie-banner.js index ee3044b545802..10afcfd6df806 100644 --- a/website/assets/js/cookie-banner.js +++ b/website/assets/js/cookie-banner.js @@ -16,7 +16,7 @@ window.addEventListener('load', function () { divB.style = "position:fixed; bottom:0px; right:0px; width:100%;"; document.body.appendChild(divA); document.body.appendChild(divB); - + // update Cookie link this.setTimeout(function () { const banner = document.getElementById('consent-banner'); @@ -25,7 +25,7 @@ window.addEventListener('load', function () { prefsElement.className = cookieLink.className; if (banner) { - // listen for click and remove banner to avoid interfering with + // listen for click and remove banner to avoid interfering with document.addEventListener('click', function (event) { const targetElement = event.target; if (targetElement.matches('#truste-consent-required') || targetElement.matches('#truste-consent-button')) { @@ -38,4 +38,4 @@ window.addEventListener('load', function () { return (cookieLink && document.getElementById('teconsent').innerHTML.length > 0) ? cookieLink.replaceWith(prefsElement) : false; }, 200); } -}); \ No newline at end of file +}); From 429598552d2e245b0ae0d9b83aa025c229f0bcbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:19:08 +0000 Subject: [PATCH 23/30] chore(deps): Bump webpki from 0.22.1 to 0.22.2 (#18744) * chore(deps): Bump webpki from 0.22.1 to 0.22.2 Bumps [webpki](https://github.com/briansmith/webpki) from 0.22.1 to 0.22.2. - [Commits](https://github.com/briansmith/webpki/commits) --- updated-dependencies: - dependency-name: webpki dependency-type: indirect ... Signed-off-by: dependabot[bot] * update webpki in license-tool.toml --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 8 ++++---- license-tool.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f5bfb12895ae..2edbebbcd86a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2296,7 +2296,7 @@ dependencies = [ "futures-core", "prost 0.12.1", "prost-types 0.12.1", - "tonic 0.10.1", + "tonic 0.10.2", "tracing-core 0.1.30", ] @@ -2318,7 +2318,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic 0.10.1", + "tonic 0.10.2", "tracing 0.1.37", "tracing-core 0.1.30", "tracing-subscriber", @@ -10411,9 +10411,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" +checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" dependencies = [ "ring", "untrusted", diff --git a/license-tool.toml b/license-tool.toml index 03f02d26d7711..f0e6df918efbf 100644 --- a/license-tool.toml +++ b/license-tool.toml @@ -11,7 +11,7 @@ "rustls-webpki-0.100.1" = { license = "ISC" } # `webpki` doesn't specify their license in the metadata, but the file contains the ISC terms. -"webpki-0.22.1" = { license = "ISC" } +"webpki-0.22.2" = { license = "ISC" } # `zerocopy` et al don't specify their licenses in the metadata, but the file contains the 2-clause BSD terms. "zerocopy-0.6.1" = { license = "BSD-2-Clause" } From 92d2be969e5bbc1af7f37abc417e003b03914348 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:59:28 +0000 Subject: [PATCH 24/30] chore(deps): Bump postcss from 8.4.6 to 8.4.31 in /website (#18750) Bumps [postcss](https://github.com/postcss/postcss) from 8.4.6 to 8.4.31. - [Release notes](https://github.com/postcss/postcss/releases) - [Changelog](https://github.com/postcss/postcss/blob/main/CHANGELOG.md) - [Commits](https://github.com/postcss/postcss/compare/8.4.6...8.4.31) --- updated-dependencies: - dependency-name: postcss dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 827c6b4c44fac..762165d7b6e67 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -3905,10 +3905,10 @@ nano-css@^5.3.1: stacktrace-js "^2.0.2" stylis "^4.0.6" -nanoid@^3.2.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.0.tgz#5906f776fd886c66c24f3653e0c46fcb1d4ad6b0" - integrity sha512-JzxqqT5u/x+/KOFSd7JP15DOo9nOoHpx6DYatqIHUW2+flybkm+mdcraotSQR5WcnZr+qhGVh8Ted0KdfSMxlg== +nanoid@^3.3.6: + version "3.3.6" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" + integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== negotiator@0.6.3: version "0.6.3" @@ -4246,11 +4246,11 @@ postcss-value-parser@^4.0.0, postcss-value-parser@^4.1.0, postcss-value-parser@^ integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== postcss@^8.1.6, postcss@^8.3.5: - version "8.4.6" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.6.tgz#c5ff3c3c457a23864f32cb45ac9b741498a09ae1" - integrity sha512-OovjwIzs9Te46vlEx7+uXB0PLijpwjXGKXjVGGPIGubGpq7uh5Xgf6D6FiJ/SzJMBosHDp6a2hiXOS97iBXcaA== + version "8.4.31" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d" + integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== dependencies: - nanoid "^3.2.0" + nanoid "^3.3.6" picocolors "^1.0.0" source-map-js "^1.0.2" From 4d98fdfa2131a216283bf73976514efb4de6241a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:00:34 +0000 Subject: [PATCH 25/30] chore(deps): Bump clap from 4.4.5 to 4.4.6 (#18715) Bumps [clap](https://github.com/clap-rs/clap) from 4.4.5 to 4.4.6. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.4.5...v4.4.6) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 36 ++++++++++++++++---------------- Cargo.toml | 2 +- lib/vector-api-client/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-vrl/cli/Cargo.toml | 2 +- lib/vector-vrl/tests/Cargo.toml | 2 +- vdev/Cargo.toml | 2 +- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2edbebbcd86a9..af493db5ebf8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,9 +172,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle 1.0.0", "anstyle-parse", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle 1.0.0", "windows-sys 0.48.0", @@ -2061,9 +2061,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -2075,15 +2075,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eef05769009513df2eb1c3b4613e7fad873a14c600ff025b08f250f59fee7de" dependencies = [ - "clap 4.4.5", + "clap 4.4.6", "log", ] [[package]] name = "clap_builder" -version = "4.4.5" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle 1.0.0", @@ -2098,7 +2098,7 @@ version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3ae8ba90b9d8b007efe66e55e48fb936272f5ca00349b5b0e89877520d35ea7" dependencies = [ - "clap 4.4.5", + "clap 4.4.6", ] [[package]] @@ -2427,7 +2427,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.5", + "clap 4.4.6", "criterion-plot", "futures 0.3.28", "is-terminal", @@ -9582,7 +9582,7 @@ dependencies = [ "atty", "cached", "chrono", - "clap 4.4.5", + "clap 4.4.6", "clap-verbosity-flag", "clap_complete", "confy", @@ -9659,7 +9659,7 @@ dependencies = [ "bytesize", "chrono", "cidr-utils", - "clap 4.4.5", + "clap 4.4.6", "codecs", "colored", "console-subscriber", @@ -9820,7 +9820,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "clap 4.4.5", + "clap 4.4.6", "futures 0.3.28", "graphql_client", "indoc", @@ -9843,7 +9843,7 @@ dependencies = [ "async-trait", "bytecheck", "bytes 1.5.0", - "clap 4.4.5", + "clap 4.4.6", "crc32fast", "criterion", "crossbeam-queue", @@ -10063,7 +10063,7 @@ dependencies = [ name = "vector-vrl-cli" version = "0.1.0" dependencies = [ - "clap 4.4.5", + "clap 4.4.6", "vector-vrl-functions", "vrl", ] @@ -10082,7 +10082,7 @@ dependencies = [ "ansi_term", "chrono", "chrono-tz", - "clap 4.4.5", + "clap 4.4.6", "enrichment", "glob", "prettydiff", @@ -10143,7 +10143,7 @@ dependencies = [ "chrono", "chrono-tz", "cidr-utils", - "clap 4.4.5", + "clap 4.4.6", "codespan-reporting", "community-id", "crypto_secretbox", diff --git a/Cargo.toml b/Cargo.toml index feae5782770f8..1b2ea3a026e3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -255,7 +255,7 @@ bytes = { version = "1.5.0", default-features = false, features = ["serde"] } bytesize = { version = "1.3.0", default-features = false } chrono = { version = "0.4.31", default-features = false, features = ["serde"] } cidr-utils = { version = "0.5.11", default-features = false } -clap = { version = "4.4.3", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +clap = { version = "4.4.6", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } colored = { version = "2.0.4", default-features = false } csv = { version = "1.2", default-features = false } derivative = { version = "2.2.0", default-features = false } diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 1591f6eb67250..d5f65276c8fbe 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -30,7 +30,7 @@ tokio-tungstenite = { version = "0.20.1", default-features = false, features = [ # External libs chrono = { version = "0.4.31", default-features = false, features = ["serde"] } -clap = { version = "4.4.3", default-features = false, features = ["derive"] } +clap = { version = "4.4.6", default-features = false, features = ["derive"] } url = { version = "2.4.1", default-features = false } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } indoc = { version = "2.0.4", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 6a434d8fede85..385d565ce0085 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -32,7 +32,7 @@ vector-config-macros = { path = "../vector-config-macros", default-features = fa vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of", "serde"] } [dev-dependencies] -clap = "4.4.3" +clap = "4.4.6" criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.8" hdrhistogram = "7.5.2" diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 0900966334e0d..5e06e58ca9c3b 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -7,6 +7,6 @@ publish = false license = "MPL-2.0" [dependencies] -clap = { version = "4.4.3", features = ["derive"] } +clap = { version = "4.4.6", features = ["derive"] } vector-vrl-functions = { path = "../functions" } vrl.workspace = true diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 17b7772e3e26f..672a60522618c 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -13,7 +13,7 @@ vector-vrl-functions = { path = "../../vector-vrl/functions" } ansi_term = "0.12" chrono = "0.4" chrono-tz = "0.8" -clap = { version = "4.4.3", features = ["derive"] } +clap = { version = "4.4.6", features = ["derive"] } glob = "0.3" prettydiff = "0.6" regex = "1" diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 2d41a85e01783..c547d4982b64d 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -12,7 +12,7 @@ anyhow = "1.0.75" atty = "0.2.14" cached = "0.45.1" chrono = { version = "0.4.31", default-features = false, features = ["serde", "clock"] } -clap = { version = "4.4.3", features = ["derive"] } +clap = { version = "4.4.6", features = ["derive"] } clap-verbosity-flag = "2.0.1" clap_complete = "4.4.3" confy = "0.5.1" From 047c7729f65bf741c7180ffea6f0af2d6723cb8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:43:19 +0000 Subject: [PATCH 26/30] chore(deps): Bump async-nats from 0.32.0 to 0.32.1 (#18735) Bumps [async-nats](https://github.com/nats-io/nats.rs) from 0.32.0 to 0.32.1. - [Release notes](https://github.com/nats-io/nats.rs/releases) - [Commits](https://github.com/nats-io/nats.rs/compare/async-nats/v0.32.0...async-nats/v0.32.1) --- updated-dependencies: - dependency-name: async-nats dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af493db5ebf8d..4063f01e396d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "async-nats" -version = "0.32.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2ea11ebe42f65b91c125042bdf8cfb0cccbd344c75e64b98fa3177040e0de9" +checksum = "0e45b67ea596bb94741ef15ba1d90b72c92bdc07553d8033734cb620a2b39f1c" dependencies = [ "base64 0.21.4", "bytes 1.5.0", diff --git a/Cargo.toml b/Cargo.toml index 1b2ea3a026e3d..e4042a8266b20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -292,7 +292,7 @@ lru = { version = "0.11.1", default-features = false, optional = true } maxminddb = { version = "0.23.0", default-features = false, optional = true } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "2.7.0", default-features = false, features = ["tokio-runtime"], optional = true } -async-nats = { version = "0.32.0", default-features = false, optional = true } +async-nats = { version = "0.32.1", default-features = false, optional = true } nkeys = { version = "0.3.1", default-features = false, optional = true } nom = { version = "7.1.3", default-features = false, optional = true } notify = { version = "6.1.1", default-features = false, features = ["macos_fsevent"] } From c7482d059e6c590fa719ee23bc55849fd68d605d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:43:57 +0000 Subject: [PATCH 27/30] chore(deps): Bump memchr from 2.6.3 to 2.6.4 (#18736) Bumps [memchr](https://github.com/BurntSushi/memchr) from 2.6.3 to 2.6.4. - [Commits](https://github.com/BurntSushi/memchr/compare/2.6.3...2.6.4) --- updated-dependencies: - dependency-name: memchr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4063f01e396d4..b377d091aa2da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5152,9 +5152,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" From f47df40104aaa60c637bce6ec06a6313de268199 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:46:43 +0000 Subject: [PATCH 28/30] chore(deps): Bump indexmap from 2.0.1 to 2.0.2 (#18737) Bumps [indexmap](https://github.com/bluss/indexmap) from 2.0.1 to 2.0.2. - [Changelog](https://github.com/bluss/indexmap/blob/master/RELEASES.md) - [Commits](https://github.com/bluss/indexmap/compare/2.0.1...2.0.2) --- updated-dependencies: - dependency-name: indexmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/prometheus-parser/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b377d091aa2da..310990f41339d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,7 +500,7 @@ dependencies = [ "fnv", "futures-util", "http", - "indexmap 2.0.1", + "indexmap 2.0.2", "mime", "multer", "num-traits", @@ -590,7 +590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86046bbced96c0fab3ff5d2b3c769c0c55b0b3a7d67f9e2f2044f349f2e7d501" dependencies = [ "bytes 1.5.0", - "indexmap 2.0.1", + "indexmap 2.0.2", "serde", "serde_json", ] @@ -3373,7 +3373,7 @@ dependencies = [ "flate2", "futures 0.3.28", "glob", - "indexmap 2.0.1", + "indexmap 2.0.2", "libc", "quickcheck", "scan_fmt", @@ -4390,9 +4390,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", "hashbrown 0.14.1", @@ -6602,7 +6602,7 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "nom", "num_enum 0.7.0", "prost 0.12.1", @@ -7838,7 +7838,7 @@ version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -7925,7 +7925,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.0.1", + "indexmap 2.0.2", "serde", "serde_json", "serde_with_macros 3.3.0", @@ -7974,7 +7974,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -8903,7 +8903,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca676d9ba1a322c1b64eb8045a5ec5c0cfb0c9d08e15e9ff622589ad5221c8fe" dependencies = [ - "indexmap 2.0.1", + "indexmap 2.0.2", "serde", "serde_spanned", "toml_datetime", @@ -9590,7 +9590,7 @@ dependencies = [ "dunce", "glob", "hex", - "indexmap 2.0.1", + "indexmap 2.0.2", "indicatif", "itertools 0.11.0", "log", @@ -9697,7 +9697,7 @@ dependencies = [ "hyper", "hyper-openssl", "hyper-proxy", - "indexmap 2.0.1", + "indexmap 2.0.2", "indoc", "infer 0.15.0", "inventory", @@ -9889,7 +9889,7 @@ dependencies = [ "crossbeam-utils", "derivative", "futures 0.3.28", - "indexmap 2.0.1", + "indexmap 2.0.2", "metrics", "nom", "ordered-float 4.1.0", @@ -9920,7 +9920,7 @@ dependencies = [ "chrono-tz", "encoding_rs", "http", - "indexmap 2.0.1", + "indexmap 2.0.2", "inventory", "no-proxy", "num-traits", @@ -9991,7 +9991,7 @@ dependencies = [ "headers", "http", "hyper-proxy", - "indexmap 2.0.1", + "indexmap 2.0.2", "metrics", "metrics-tracing-context", "metrics-util", @@ -10158,7 +10158,7 @@ dependencies = [ "hex", "hmac", "hostname", - "indexmap 2.0.1", + "indexmap 2.0.2", "indoc", "itertools 0.11.0", "lalrpop", diff --git a/Cargo.toml b/Cargo.toml index e4042a8266b20..f050eeb670672 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -280,7 +280,7 @@ http-body = { version = "0.4.5", default-features = false } hyper = { version = "0.14.27", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] } hyper-openssl = { version = "0.9.2", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } -indexmap = { version = "~2.0.1", default-features = false, features = ["serde", "std"] } +indexmap = { version = "~2.0.2", default-features = false, features = ["serde", "std"] } infer = { version = "0.15.0", default-features = false, optional = true} indoc = { version = "2.0.4", default-features = false } inventory = { version = "0.3.12", default-features = false } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 64fefc8e16bee..e854b24677148 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -39,7 +39,7 @@ default-features = false features = [] [dependencies.indexmap] -version = "~2.0.1" +version = "~2.0.2" default-features = false features = ["serde"] diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index 8c2cad878944e..ee863d710afdd 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -9,7 +9,7 @@ license = "MPL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -indexmap = "~2.0.1" +indexmap = "~2.0.2" nom = "7.1.3" num_enum = "0.7.0" prost = "0.12" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 18eb337ae2704..53c12c2b9890a 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,7 +48,7 @@ chrono = { version = "0.4", default-features = false, optional = true, features crossbeam-utils = { version = "0.8.16", default-features = false } derivative = { version = "2.2.0", default-features = false } futures = { version = "0.3.28", default-features = false, features = ["std"] } -indexmap = { version = "~2.0.1", default-features = false, features = ["std"] } +indexmap = { version = "~2.0.2", default-features = false, features = ["std"] } metrics = "0.21.1" nom = { version = "7", optional = true } ordered-float = { version = "4.1.0", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 9bd5d5a64bb09..e1f88dfbc9084 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -23,7 +23,7 @@ futures-util = { version = "0.3.28", default-features = false, features = ["std" headers = { version = "0.3.9", default-features = false } http = { version = "0.2.9", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } -indexmap = { version = "~2.0.1", default-features = false, features = ["serde", "std"] } +indexmap = { version = "~2.0.2", default-features = false, features = ["serde", "std"] } lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics = "0.21.1" metrics-tracing-context = { version = "0.14.0", default-features = false } From 8e2032c407985a2dd77b7550b62d5fcfd04399d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 18:39:25 +0000 Subject: [PATCH 29/30] chore(ci): Bump aws-actions/amazon-ecr-login from 1 to 2 (#18752) Bumps [aws-actions/amazon-ecr-login](https://github.com/aws-actions/amazon-ecr-login) from 1 to 2. - [Release notes](https://github.com/aws-actions/amazon-ecr-login/releases) - [Changelog](https://github.com/aws-actions/amazon-ecr-login/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/amazon-ecr-login/compare/v1...v2) --- updated-dependencies: - dependency-name: aws-actions/amazon-ecr-login dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/regression.yml | 6 +++--- .github/workflows/workload_checks.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 089e9c9097914..387750b55ddf9 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -402,7 +402,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 - name: Docker Login to ECR uses: docker/login-action@v3 @@ -440,7 +440,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 - name: Docker Login to ECR uses: docker/login-action@v3 @@ -486,7 +486,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 - name: Download SMP binary run: | diff --git a/.github/workflows/workload_checks.yml b/.github/workflows/workload_checks.yml index 3610c78bdfa15..76d336ebe2a99 100644 --- a/.github/workflows/workload_checks.yml +++ b/.github/workflows/workload_checks.yml @@ -104,7 +104,7 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 + uses: aws-actions/amazon-ecr-login@v2 - name: Download SMP binary run: | From 23745f213f18667548c204133bfd09d55b8ff8c5 Mon Sep 17 00:00:00 2001 From: Devin Ford Date: Tue, 3 Oct 2023 16:52:37 -0400 Subject: [PATCH 30/30] chore(website): Set download page dropdown to latest version (#18758) * feat: set to latest version on page load * feat: add a local dev with hot reload for development * chore: revert makefile changes --- website/layouts/download/section.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/layouts/download/section.html b/website/layouts/download/section.html index 6f4b403e9b961..2ef6c3c19ddbf 100644 --- a/website/layouts/download/section.html +++ b/website/layouts/download/section.html @@ -6,7 +6,7 @@ {{ $releases := site.Data.docs.releases }} {{ $versions := site.Data.docs.versions }} {{ $latest := index $versions 0 }} -
+
{{ partial "hero.html" . }}