chore(dev): Wrap vector-core in vector-lib (#18934)

* chore(dev): Wrap `vector-core` in `vector-lib`

This is where things get a little more interesting. Both `vector-common` and
`vector-core` export a `config` module, so we need to create a virtual module
pulling in the relevant symbols. I have made those imports explicit so as to
eliminate the ambiguity of which symbol comes from which package.

* Feature flag fixes

Cargo.toml

@@ -144,8 +144,7 @@ vector-buffers = { path = "lib/vector-buffers", default-features = false }
 vector-config = { path = "lib/vector-config" }
 vector-config-common = { path = "lib/vector-config-common" }
 vector-config-macros = { path = "lib/vector-config-macros" }
-vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl"] }
-vector-lib = { path = "lib/vector-lib", default-features = false }
+vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
 vector-stream = { path = "lib/vector-stream" }
 vector-vrl-functions = { path = "lib/vector-vrl/functions" }
 loki-logproto = { path = "lib/loki-logproto", optional = true }
@@ -377,7 +376,7 @@ test-generator = "0.3.1"
 tokio = { version = "1.33.0", features = ["test-util"] }
 tokio-test = "0.4.3"
 tower-test = "0.4.0"
-vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] }
+vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl", "test"] }
 vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] }
 
 wiremock = "0.5.19"
@@ -438,7 +437,7 @@ api = [
   "dep:async-graphql",
   "dep:async-graphql-warp",
   "dep:base64",
-  "vector-core/api",
+  "vector-lib/api",
 ]
 
 # API client
@@ -447,7 +446,7 @@ api-client = [
   "dep:num-format",
   "dep:number_prefix",
   "dep:ratatui",
-  "vector-core/api",
+  "vector-lib/api",
   "dep:vector-api-client",
 ]
 
@@ -608,7 +607,7 @@ transforms-aws_ec2_metadata = ["dep:arc-swap"]
 transforms-dedupe = ["dep:lru"]
 transforms-filter = []
 transforms-log_to_metric = []
-transforms-lua = ["dep:mlua", "vector-core/lua"]
+transforms-lua = ["dep:mlua", "vector-lib/lua"]
 transforms-metric_to_log = []
 transforms-pipelines = ["transforms-filter", "transforms-route"]
 transforms-reduce = []

lib/vector-lib/Cargo.toml

@@ -7,3 +7,10 @@ publish = false
 
 [dependencies]
 vector-common = { path = "../vector-common", default-features = false }
+vector-core = { path = "../vector-core", default-features = false }
+
+[features]
+api = ["vector-core/api"]
+lua = ["vector-core/lua"]
+test = ["vector-core/test"]
+vrl = ["vector-core/vrl"]

lib/vector-lib/src/lib.rs

@@ -1,6 +1,25 @@
 pub use vector_common::{
-    assert_event_data_eq, btreemap, byte_size_of, byte_size_of::ByteSizeOf, config, conversion,
+    assert_event_data_eq, btreemap, byte_size_of, byte_size_of::ByteSizeOf, conversion,
     encode_logfmt, finalization, finalizer, impl_event_data_eq, internal_event, json_size,
     registered_event, request_metadata, sensitive_string, shutdown, trigger, Error, Result,
     TimeZone,
 };
+#[cfg(feature = "vrl")]
+pub use vector_core::compile_vrl;
+#[cfg(feature = "test")]
+pub use vector_core::event_test_util;
+pub use vector_core::{
+    buckets, buffers, default_data_dir, event, fanout, metric_tags, metrics, partition, quantiles,
+    samples, schema, serde, sink, source, tcp, tls, transform, update_counter,
+    EstimatedJsonEncodedSizeOf,
+};
+
+pub mod config {
+    pub use vector_common::config::ComponentKey;
+    pub use vector_core::config::{
+        clone_input_definitions, init_log_schema, init_telemetry, log_schema, proxy, telemetry,
+        AcknowledgementsConfig, DataType, GlobalOptions, Input, LegacyKey, LogNamespace, LogSchema,
+        OutputId, SourceAcknowledgementsConfig, SourceOutput, Tags, Telemetry, TransformOutput,
+        MEMORY_BUFFER_DEFAULT_MAX_EVENTS,
+    };
+}

src/api/schema/components/mod.rs

@@ -11,7 +11,7 @@ use std::{
 use async_graphql::{Enum, InputObject, Interface, Object, Subscription};
 use once_cell::sync::Lazy;
 use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt};
-use vector_core::internal_event::DEFAULT_OUTPUT;
+use vector_lib::internal_event::DEFAULT_OUTPUT;
 
 use crate::{
     api::schema::{

src/api/schema/metrics/output.rs

@@ -1,5 +1,5 @@
 use async_graphql::Object;
-use vector_core::config::ComponentKey;
+use vector_lib::config::ComponentKey;
 
 use super::{by_component_key, sum_metrics, SentEventsTotal};
 use crate::event::Metric;

src/codecs/decoding/config.rs

@@ -1,6 +1,6 @@
 use codecs::decoding::{DeserializerConfig, FramingConfig};
 use serde::{Deserialize, Serialize};
-use vector_core::config::LogNamespace;
+use vector_lib::config::LogNamespace;
 
 use crate::codecs::Decoder;
 

src/codecs/decoding/decoder.rs

@@ -4,7 +4,7 @@ use codecs::decoding::{
     NewlineDelimitedDecoder,
 };
 use smallvec::SmallVec;
-use vector_core::config::LogNamespace;
+use vector_lib::config::LogNamespace;
 
 use crate::{
     event::Event,

src/codecs/encoding/encoder.rs

@@ -187,7 +187,7 @@ mod tests {
     use codecs::encoding::BoxedFramingError;
     use futures_util::{SinkExt, StreamExt};
     use tokio_util::codec::FramedWrite;
-    use vector_core::event::LogEvent;
+    use vector_lib::event::LogEvent;
 
     use super::*;
 

src/codecs/encoding/transformer.rs

@@ -9,8 +9,8 @@ use lookup::{event_path, PathPrefix};
 use ordered_float::NotNan;
 use serde::{Deserialize, Deserializer};
 use vector_config::configurable_component;
-use vector_core::event::{LogEvent, MaybeAsLogMut};
-use vector_core::schema::meaning;
+use vector_lib::event::{LogEvent, MaybeAsLogMut};
+use vector_lib::schema::meaning;
 use vrl::path::OwnedValuePath;
 use vrl::value::Value;
 
@@ -267,8 +267,8 @@ pub enum TimestampFormat {
 mod tests {
     use indoc::indoc;
     use lookup::path::parse_target_path;
-    use vector_core::config::{log_schema, LogNamespace};
     use vector_lib::btreemap;
+    use vector_lib::config::{log_schema, LogNamespace};
     use vrl::value::Kind;
 
     use crate::config::schema;

src/common/datadog.rs

@@ -4,7 +4,7 @@
 #![allow(unreachable_pub)]
 use serde::{Deserialize, Serialize};
 use vector_config::configurable_component;
-use vector_core::event::DatadogMetricOriginMetadata;
+use vector_lib::event::DatadogMetricOriginMetadata;
 
 pub const DD_US_SITE: &str = "datadoghq.com";
 pub const DD_EU_SITE: &str = "datadoghq.eu";

src/components/validation/resources/event.rs

@@ -8,7 +8,7 @@ use codecs::{
     encoding, JsonSerializer, LengthDelimitedEncoder, LogfmtSerializer, MetricTagValues,
     NewlineDelimitedEncoder,
 };
-use vector_core::event::{Event, LogEvent};
+use vector_lib::event::{Event, LogEvent};
 
 /// A test case event for deserialization from yaml file.
 /// This is an intermediary step to TestEvent.

src/components/validation/resources/http.rs

@@ -21,7 +21,7 @@ use tokio::{
 use tokio_util::codec::Decoder;
 
 use crate::components::validation::sync::{Configuring, TaskCoordinator};
-use vector_core::event::Event;
+use vector_lib::event::Event;
 
 use super::{encode_test_event, ResourceCodec, ResourceDirection, TestEvent};
 

src/components/validation/resources/mod.rs

@@ -9,7 +9,7 @@ use codecs::{
     BytesEncoder,
 };
 use tokio::sync::mpsc;
-use vector_core::{config::DataType, event::Event};
+use vector_lib::{config::DataType, event::Event};
 
 use crate::codecs::{Decoder, DecodingConfig, Encoder, EncodingConfig, EncodingConfigWithFraming};
 

src/components/validation/runner/io.rs

@@ -9,8 +9,8 @@ use tonic::{
     Status,
 };
 use tower::Service;
-use vector_core::{event::Event, tls::MaybeTlsSettings};
 use vector_lib::shutdown::ShutdownSignal;
+use vector_lib::{event::Event, tls::MaybeTlsSettings};
 
 use crate::{
     components::validation::{

src/components/validation/runner/mod.rs

@@ -19,7 +19,7 @@ use tokio::{
 use tokio_util::codec::Encoder as _;
 
 use codecs::encoding;
-use vector_core::{event::Event, EstimatedJsonEncodedSizeOf};
+use vector_lib::{event::Event, EstimatedJsonEncodedSizeOf};
 
 use crate::{
     codecs::Encoder,

src/components/validation/runner/telemetry.rs

@@ -1,7 +1,7 @@
 use std::time::Duration;
 
 use tokio::{select, sync::mpsc, task::JoinHandle};
-use vector_core::event::Event;
+use vector_lib::event::Event;
 
 use crate::{
     components::validation::{

src/components/validation/validators/component_spec/mod.rs

@@ -1,6 +1,6 @@
 mod sources;
 
-use vector_core::event::{Event, Metric};
+use vector_lib::event::{Event, Metric};
 
 use crate::components::validation::{ComponentType, RunnerMetrics, TestCaseExpectation, TestEvent};
 
@@ -124,7 +124,7 @@ fn filter_events_by_metric_and_component<'a>(
     let metrics: Vec<&Metric> = telemetry_events
         .iter()
         .flat_map(|e| {
-            if let vector_core::event::Event::Metric(m) = e {
+            if let vector_lib::event::Event::Metric(m) = e {
                 Some(m)
             } else {
                 None

src/components/validation/validators/component_spec/sources.rs

@@ -1,6 +1,6 @@
 use std::fmt::{Display, Formatter};
 
-use vector_core::event::{Event, MetricKind};
+use vector_lib::event::{Event, MetricKind};
 
 use crate::components::validation::RunnerMetrics;
 
@@ -68,14 +68,14 @@ pub fn validate_sources(
 
 fn sum_counters(
     metric_name: &SourceMetricType,
-    metrics: &[&vector_core::event::Metric],
+    metrics: &[&vector_lib::event::Metric],
 ) -> Result<u64, Vec<String>> {
     let mut sum: f64 = 0.0;
     let mut errs = Vec::new();
 
     for m in metrics {
         match m.value() {
-            vector_core::event::MetricValue::Counter { value } => {
+            vector_lib::event::MetricValue::Counter { value } => {
                 if let MetricKind::Absolute = m.data().kind {
                     sum = *value;
                 } else {

src/components/validation/validators/mod.rs

@@ -2,7 +2,7 @@ mod component_spec;
 
 pub use self::component_spec::ComponentSpecValidator;
 
-use vector_core::event::Event;
+use vector_lib::event::Event;
 
 use super::{ComponentType, RunnerMetrics, TestCaseExpectation, TestEvent};
 

src/conditions/datadog_search.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 
 use bytes::Bytes;
 use vector_config::configurable_component;
-use vector_core::event::{Event, LogEvent, Value};
+use vector_lib::event::{Event, LogEvent, Value};
 use vrl::datadog_filter::{
     build_matcher,
     regex::{wildcard_regex, word_regex},
@@ -325,7 +325,7 @@ where
 #[cfg(test)]
 mod test {
     use serde_json::json;
-    use vector_core::event::Event;
+    use vector_lib::event::Event;
     use vrl::datadog_filter::{build_matcher, Filter, Resolver};
     use vrl::datadog_search_syntax::parse;
 

src/conditions/vrl.rs

@@ -1,5 +1,5 @@
 use vector_config::configurable_component;
-use vector_core::compile_vrl;
+use vector_lib::compile_vrl;
 use vector_lib::TimeZone;
 use vrl::compiler::runtime::{Runtime, RuntimeResult, Terminate};
 use vrl::compiler::{CompilationResult, CompileConfig, Program, TypeState, VrlRuntime};
@@ -170,7 +170,7 @@ impl Conditional for Vrl {
 
 #[cfg(test)]
 mod test {
-    use vector_core::metric_tags;
+    use vector_lib::metric_tags;
 
     use super::*;
     use crate::{

src/config/builder.rs

@@ -6,7 +6,7 @@ use indexmap::IndexMap;
 #[cfg(feature = "enterprise")]
 use serde_json::Value;
 use vector_config::configurable_component;
-use vector_core::config::GlobalOptions;
+use vector_lib::config::GlobalOptions;
 
 use crate::{enrichment_tables::EnrichmentTables, providers::Providers, secrets::SecretBackends};
 

src/config/compiler.rs

@@ -198,7 +198,7 @@ fn expand_globs_inner(inputs: &mut Inputs<String>, id: &str, candidates: &IndexS
 mod test {
     use super::*;
     use crate::test_util::mock::{basic_sink, basic_source, basic_transform};
-    use vector_core::config::ComponentKey;
+    use vector_lib::config::ComponentKey;
 
     #[test]
     fn glob_expansion() {

src/config/enrichment_table.rs

@@ -1,7 +1,7 @@
 use async_trait::async_trait;
 use enum_dispatch::enum_dispatch;
 use vector_config::{configurable_component, NamedComponent};
-use vector_core::config::GlobalOptions;
+use vector_lib::config::GlobalOptions;
 
 use crate::enrichment_tables::EnrichmentTables;
 

src/config/enterprise.rs

@@ -14,7 +14,7 @@ use tokio::{
     time::{sleep, Duration},
 };
 use url::{ParseError, Url};
-use vector_core::config::proxy::ProxyConfig;
+use vector_lib::config::proxy::ProxyConfig;
 
 use super::{
     load_source_from_paths, process_paths, ComponentKey, Config, ConfigPath, OutputId, SinkOuter,
@@ -832,7 +832,7 @@ mod test {
     use http::StatusCode;
     use indexmap::IndexMap;
     use tokio::time::sleep;
-    use vector_core::config::proxy::ProxyConfig;
+    use vector_lib::config::proxy::ProxyConfig;
     use vrl::btreemap;
     use vrl::compiler::state::ExternalEnv;
     use vrl::compiler::{compile, compile_with_external, CompileConfig};

src/config/graph.rs

@@ -367,7 +367,7 @@ impl Graph {
 #[cfg(test)]
 mod test {
     use similar_asserts::assert_eq;
-    use vector_core::schema::Definition;
+    use vector_lib::schema::Definition;
 
     use super::*;
 

src/config/id.rs

@@ -1,7 +1,7 @@
 use std::ops::Deref;
 
 use vector_config::configurable_component;
-pub use vector_core::config::ComponentKey;
+pub use vector_lib::config::ComponentKey;
 
 /// A list of upstream [source][sources] or [transform][transforms] IDs.
 ///