Make filter for tracing-opentelemetry and tracing-chrome configurable (…

…#1496)

aggregator/src/trace.rs

@@ -3,9 +3,10 @@
 use atty::{self, Stream};
 use serde::{Deserialize, Serialize};
 use std::{collections::HashMap, net::SocketAddr};
+use tracing::Level;
 use tracing_chrome::{ChromeLayerBuilder, TraceStyle};
 use tracing_log::LogTracer;
-use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer, Registry};
+use tracing_subscriber::{filter::FromEnvError, layer::SubscriberExt, EnvFilter, Layer, Registry};
 
 #[cfg(feature = "otlp")]
 use {
@@ -17,7 +18,6 @@ use {
     opentelemetry_semantic_conventions::resource::SERVICE_NAME,
     std::str::FromStr,
     tonic::metadata::{MetadataKey, MetadataMap, MetadataValue},
-    tracing_subscriber::filter::{LevelFilter, Targets},
 };
 
 /// Errors from initializing trace subscriber.
@@ -36,6 +36,8 @@ pub enum Error {
     #[cfg(feature = "otlp")]
     #[error(transparent)]
     TonicMetadataValue(#[from] tonic::metadata::errors::InvalidMetadataValue),
+    #[error("bad log/trace filter: {0}")]
+    FromEnv(#[from] FromEnvError),
     #[error("{0}")]
     Other(&'static str),
 }
@@ -113,6 +115,15 @@ fn base_layer<S>() -> tracing_subscriber::fmt::Layer<S> {
         .with_line_number(true)
 }
 
+/// Construct a filter to be used with tracing-opentelemetry and tracing-chrome, based on the
+/// contents of the `RUST_TRACE` environment variable.
+fn make_trace_filter() -> Result<EnvFilter, FromEnvError> {
+    EnvFilter::builder()
+        .with_default_directive(Level::INFO.into())
+        .with_env_var("RUST_TRACE")
+        .from_env()
+}
+
 /// Configures and installs a tracing subscriber, to capture events logged with
 /// [`tracing::info`] and the like. Captured events are written to stdout, with
 /// formatting affected by the provided [`TraceConfiguration`].
@@ -123,7 +134,7 @@ pub fn install_trace_subscriber(config: &TraceConfiguration) -> Result<TraceGuar
 
     // Configure filters with RUST_LOG env var. Format discussed at
     // https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html
-    let stdout_filter = EnvFilter::from_default_env();
+    let stdout_filter = EnvFilter::builder().from_env()?;
 
     let mut layers = Vec::new();
     match (
@@ -199,16 +210,10 @@ pub fn install_trace_subscriber(config: &TraceConfiguration) -> Result<TraceGuar
             ]))))
             .install_batch(opentelemetry::runtime::Tokio)?;
 
-        // Filter out some spans from h2, internal to the OTLP exporter (via tonic). These spans
-        // would otherwise drown out root spans from the application.
-        let filter = Targets::new()
-            .with_default(LevelFilter::TRACE)
-            .with_target("h2", LevelFilter::OFF);
-
         let telemetry = tracing_opentelemetry::layer()
             .with_threads(true)
             .with_tracer(tracer)
-            .with_filter(filter);
+            .with_filter(make_trace_filter()?);
         layers.push(telemetry.boxed());
     }
 
@@ -227,7 +232,7 @@ pub fn install_trace_subscriber(config: &TraceConfiguration) -> Result<TraceGuar
             .include_args(true)
             .build();
         chrome_guard = Some(guard);
-        layers.push(layer.boxed());
+        layers.push(layer.with_filter(make_trace_filter()?).boxed());
     }
 
     let subscriber = Registry::default().with(layers);

core/src/test_util/mod.rs

@@ -144,7 +144,7 @@ pub fn roundtrip_encoding<T: Serialize + DeserializeOwned + Debug + Eq>(value: T
 pub fn install_test_trace_subscriber() {
     static INSTALL_TRACE_SUBSCRIBER: Once = Once::new();
     INSTALL_TRACE_SUBSCRIBER.call_once(|| {
-        let stdout_filter = EnvFilter::from_default_env();
+        let stdout_filter = EnvFilter::builder().from_env().unwrap();
         let layer = tracing_subscriber::fmt::layer()
             .with_thread_ids(true)
             .with_level(true)

docs/CONFIGURING_TRACING.md

@@ -3,6 +3,12 @@
 Tracing spans from Janus components can be exported to distributed tracing
 systems through the OpenTelemetry SDK, and various exporters.
 
+Verbosity of traces can be controlled by setting the `RUST_TRACE` environment
+variable to a [filter][EnvFilter], just as with `RUST_LOG` for log output. By
+default, all spans and events of severity `INFO` or higher will be exported.
+
+[EnvFilter]: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/struct.EnvFilter.html
+
 ## Jaeger
 
 [Jaeger](https://www.jaegertracing.io/) is a software stack that stores,

interop_binaries/src/lib.rs

@@ -318,7 +318,7 @@ impl From<Task> for AggregatorAddTaskRequest {
 }
 
 pub fn install_tracing_subscriber() -> anyhow::Result<()> {
-    let stdout_filter = EnvFilter::from_default_env();
+    let stdout_filter = EnvFilter::builder().from_env()?;
     let layer = tracing_subscriber::fmt::layer()
         .with_thread_ids(true)
         .with_level(true)

tools/src/bin/collect.rs

@@ -564,7 +564,7 @@ where
 }
 
 fn install_tracing_subscriber() -> anyhow::Result<()> {
-    let stdout_filter = EnvFilter::from_default_env();
+    let stdout_filter = EnvFilter::builder().from_env()?;
     let layer = tracing_subscriber::fmt::layer()
         .with_level(true)
         .with_target(true)