chore(dev): Add wrapper for vector-buffers to vector-lib (vectord…

…otdev#18964)

Cargo.toml

@@ -139,7 +139,6 @@ prometheus-parser = { path = "lib/prometheus-parser", optional = true }
 opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true }
 tracing-limit = { path = "lib/tracing-limit" }
 vector-api-client = { path = "lib/vector-api-client", optional = true }
-vector-buffers = { path = "lib/vector-buffers", default-features = false }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
 vector-vrl-functions = { path = "lib/vector-vrl/functions" }
 loki-logproto = { path = "lib/loki-logproto", optional = true }

lib/vector-core/src/lib.rs

@@ -51,7 +51,6 @@ use std::path::PathBuf;
 #[cfg(feature = "vrl")]
 pub use crate::vrl::compile_vrl;
 
-pub use vector_buffers as buffers;
 #[cfg(any(test, feature = "test"))]
 pub use vector_common::event_test_util;
 pub use vector_common::{byte_size_of::ByteSizeOf, internal_event};

lib/vector-lib/Cargo.toml

@@ -7,6 +7,7 @@ publish = false
 
 [dependencies]
 codecs = { path = "../codecs", default-features = false }
+vector-buffers = { path = "../vector-buffers", default-features = false }
 vector-common = { path = "../vector-common" }
 vector-config = { path = "../vector-config" }
 vector-core = { path = "../vector-core", default-features = false }

lib/vector-lib/src/lib.rs

@@ -1,4 +1,5 @@
 pub use codecs;
+pub use vector_buffers as buffers;
 pub use vector_common::{
     assert_event_data_eq, btreemap, byte_size_of, byte_size_of::ByteSizeOf, conversion,
     encode_logfmt, finalization, finalizer, impl_event_data_eq, internal_event, json_size,
@@ -11,9 +12,8 @@ pub use vector_core::compile_vrl;
 #[cfg(feature = "test")]
 pub use vector_core::event_test_util;
 pub use vector_core::{
-    buckets, buffers, default_data_dir, event, fanout, metric_tags, metrics, partition, quantiles,
-    samples, schema, serde, sink, source, tcp, tls, transform, update_counter,
-    EstimatedJsonEncodedSizeOf,
+    buckets, default_data_dir, event, fanout, metric_tags, metrics, partition, quantiles, samples,
+    schema, serde, sink, source, tcp, tls, transform, update_counter, EstimatedJsonEncodedSizeOf,
 };
 pub use vector_stream as stream;
 

src/api/tap.rs

@@ -10,7 +10,7 @@ use tokio::sync::{
     oneshot,
 };
 use uuid::Uuid;
-use vector_buffers::{topology::builder::TopologyBuilder, WhenFull};
+use vector_lib::buffers::{topology::builder::TopologyBuilder, WhenFull};
 
 use super::{
     schema::events::{

src/config/sink.rs

@@ -3,7 +3,7 @@ use std::cell::RefCell;
 use async_trait::async_trait;
 use dyn_clone::DynClone;
 use serde::Serialize;
-use vector_buffers::{BufferConfig, BufferType};
+use vector_lib::buffers::{BufferConfig, BufferType};
 use vector_lib::configurable::attributes::CustomAttribute;
 use vector_lib::configurable::schema::{SchemaGenerator, SchemaObject};
 use vector_lib::configurable::{

src/sinks/elasticsearch/encoder.rs

@@ -1,7 +1,7 @@
 use std::{io, io::Write};
 
 use serde::Serialize;
-use vector_buffers::EventCount;
+use vector_lib::buffers::EventCount;
 use vector_lib::{config::telemetry, event::Event, ByteSizeOf, EstimatedJsonEncodedSizeOf};
 use vector_lib::{
     internal_event::TaggedEventsSent,

src/sinks/prelude.rs

@@ -4,7 +4,7 @@
 pub use async_trait::async_trait;
 pub use futures::{future, future::BoxFuture, stream::BoxStream, FutureExt, StreamExt};
 pub use tower::{Service, ServiceBuilder};
-pub use vector_buffers::EventCount;
+pub use vector_lib::buffers::EventCount;
 pub use vector_lib::configurable::configurable_component;
 pub use vector_lib::stream::{BatcherSettings, DriverResponse};
 pub use vector_lib::{

src/source_sender/errors.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 
 use tokio::sync::mpsc;
-use vector_buffers::topology::channel::SendError;
+use vector_lib::buffers::topology::channel::SendError;
 
 use crate::event::{Event, EventArray};
 

src/source_sender/mod.rs

@@ -6,7 +6,7 @@ use chrono::Utc;
 use futures::{Stream, StreamExt};
 use metrics::{register_histogram, Histogram};
 use tracing::Span;
-use vector_buffers::topology::channel::{self, LimitedReceiver, LimitedSender};
+use vector_lib::buffers::topology::channel::{self, LimitedReceiver, LimitedSender};
 #[cfg(test)]
 use vector_lib::event::{into_event_stream, EventStatus};
 use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL};

src/sources/kafka.rs

@@ -1531,7 +1531,7 @@ mod integration_test {
     };
     use stream_cancel::{Trigger, Tripwire};
     use tokio::time::sleep;
-    use vector_buffers::topology::channel::BufferReceiver;
+    use vector_lib::buffers::topology::channel::BufferReceiver;
     use vector_lib::event::EventStatus;
     use vrl::{event_path, value};
 

src/test_util/mock/sources/basic.rs

@@ -4,7 +4,7 @@ use std::sync::{
 };
 
 use async_trait::async_trait;
-use vector_buffers::topology::channel::{limited, LimitedReceiver};
+use vector_lib::buffers::topology::channel::{limited, LimitedReceiver};
 use vector_lib::configurable::configurable_component;
 use vector_lib::{config::LogNamespace, schema::Definition};
 use vector_lib::{

src/test_util/mod.rs

@@ -34,7 +34,7 @@ use tokio_stream::wrappers::TcpListenerStream;
 #[cfg(unix)]
 use tokio_stream::wrappers::UnixListenerStream;
 use tokio_util::codec::{Encoder, FramedRead, FramedWrite, LinesCodec};
-use vector_buffers::topology::channel::LimitedReceiver;
+use vector_lib::buffers::topology::channel::LimitedReceiver;
 use vector_lib::event::{BatchNotifier, Event, EventArray, LogEvent};
 #[cfg(test)]
 use zstd::Decoder as ZstdDecoder;

src/topology/mod.rs

@@ -27,7 +27,7 @@ use std::{
 
 use futures::{Future, FutureExt};
 use tokio::sync::{mpsc, watch};
-use vector_buffers::topology::channel::{BufferReceiverStream, BufferSender};
+use vector_lib::buffers::topology::channel::{BufferReceiverStream, BufferSender};
 
 pub use self::builder::TopologyPieces;
 pub use self::controller::{ReloadOutcome, SharedTopologyController, TopologyController};

src/topology/running.rs

@@ -12,7 +12,7 @@ use tokio::{
     time::{interval, sleep_until, Duration, Instant},
 };
 use tracing::Instrument;
-use vector_buffers::topology::channel::BufferSender;
+use vector_lib::buffers::topology::channel::BufferSender;
 use vector_lib::trigger::DisabledTrigger;
 
 use super::{

src/topology/task.rs

@@ -9,7 +9,7 @@ use futures::{future::BoxFuture, FutureExt};
 use pin_project::pin_project;
 use snafu::Snafu;
 use tokio::task::JoinError;
-use vector_buffers::topology::channel::BufferReceiverStream;
+use vector_lib::buffers::topology::channel::BufferReceiverStream;
 use vector_lib::event::EventArray;
 
 use crate::{config::ComponentKey, utilization::Utilization};

src/topology/test/backpressure.rs

@@ -2,7 +2,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
 use tokio::time::Duration;
-use vector_buffers::{BufferConfig, BufferType, WhenFull};
+use vector_lib::buffers::{BufferConfig, BufferType, WhenFull};
 use vector_lib::config::MEMORY_BUFFER_DEFAULT_MAX_EVENTS;
 
 use crate::{config::Config, test_util, test_util::start_topology};

src/topology/test/mod.rs

@@ -26,7 +26,7 @@ use tokio::{
     task::yield_now,
     time::{sleep, Duration},
 };
-use vector_buffers::{BufferConfig, BufferType, WhenFull};
+use vector_lib::buffers::{BufferConfig, BufferType, WhenFull};
 use vector_lib::config::ComponentKey;
 use vector_lib::config::OutputId;
 

src/topology/test/reload.rs

@@ -7,7 +7,7 @@ use std::{
 use futures::StreamExt;
 use tokio::time::sleep;
 use tokio_stream::wrappers::UnboundedReceiverStream;
-use vector_buffers::{BufferConfig, BufferType, WhenFull};
+use vector_lib::buffers::{BufferConfig, BufferType, WhenFull};
 use vector_lib::config::ComponentKey;
 
 use crate::{