From a4b0df147f5344a29b39aa7820db456c1163a041 Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 10 Oct 2022 13:43:39 +0200 Subject: [PATCH] Add support for OpenTelemetry tracing via OTLP This allows using the GRPC endpoint of an OTLP exporter to export span-based telemetry data directly from conmon-rs. Signed-off-by: Sascha Grunert --- .github/install-deps | 2 +- .github/workflows/cross.yml | 8 - Cargo.lock | 605 +++++++++++++++++++++ Cross.toml | 2 +- conmon-rs/server/Cargo.toml | 4 + conmon-rs/server/src/config.rs | 24 +- conmon-rs/server/src/init.rs | 3 +- conmon-rs/server/src/server.rs | 59 +- contrib/tracing/docker-compose.yaml | 22 + contrib/tracing/otel-collector-config.yaml | 36 ++ 10 files changed, 746 insertions(+), 19 deletions(-) create mode 100644 contrib/tracing/docker-compose.yaml create mode 100644 contrib/tracing/otel-collector-config.yaml diff --git a/.github/install-deps b/.github/install-deps index 25ac15623d..8495432017 100755 --- a/.github/install-deps +++ b/.github/install-deps @@ -2,4 +2,4 @@ set -euo pipefail sudo apt-get update -sudo apt-get install capnproto runc +sudo apt-get install capnproto runc protobuf-compiler diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index a4d937902c..33b99137ea 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -7,7 +7,6 @@ on: env: CROSS_VERSION: v0.2.4 CARGO_TERM_COLOR: always - ACTION_MSRV_TOOLCHAIN: 1.58.1 jobs: build: strategy: @@ -31,13 +30,6 @@ jobs: ~/.cargo/git target key: ${{ runner.os }}-cross-${{matrix.arch}}-${{ hashFiles('**/Cargo.lock') }} - - name: Select Toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ env['ACTION_MSRV_TOOLCHAIN'] }} - default: true - override: true - components: rustfmt - name: Install cross-rs run: | curl -sSfL --retry 5 --retry-delay 3 \ diff --git a/Cargo.lock b/Cargo.lock index 9c40e32e0e..0ee846c987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,12 +69,44 @@ dependencies = [ "futures-lite", ] +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-task" version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" +[[package]] +name = "async-trait" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.0.0" @@ -98,6 +130,57 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e3356844c4d6a6d6467b8da2cffb4a2820be256f50a3a386c9d152bab31043" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f0c0a60006f2a293d82d571f635042a72edf927539b7685bd62d361963839b" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" @@ -262,6 +345,7 @@ dependencies = [ "capnp-rpc", "clap", "conmon-common", + "dashmap", "futures", "getset", "lazy_static", @@ -272,6 +356,8 @@ dependencies = [ "multimap", "nix 0.25.0", "notify", + "opentelemetry", + "opentelemetry-otlp", "prctl", "regex", "sendfd", @@ -285,6 +371,7 @@ dependencies = [ "tokio-fd", "tokio-util", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "tz-rs", "uuid 1.1.2", @@ -375,6 +462,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "dashmap" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c" +dependencies = [ + "cfg-if", + "num_cpus", + "parking_lot", +] + [[package]] name = "difflib" version = "0.4.0" @@ -452,6 +550,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "float-cmp" version = "0.9.0" @@ -461,6 +565,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -635,6 +745,25 @@ dependencies = [ "url", ] +[[package]] +name = "h2" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -665,6 +794,82 @@ dependencies = [ "digest", ] +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "iana-time-zone" version = "0.1.50" @@ -852,6 +1057,16 @@ version = "0.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -862,6 +1077,12 @@ dependencies = [ "serde", ] +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + [[package]] name = "memchr" version = "2.5.0" @@ -877,6 +1098,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1036,6 +1263,86 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "http", + "opentelemetry", + "opentelemetry-proto", + "prost", + "thiserror", + "tokio", + "tonic", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +dependencies = [ + "futures", + "futures-util", + "opentelemetry", + "prost", + "tonic", + "tonic-build", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "os_str_bytes" version = "6.3.0" @@ -1054,12 +1361,45 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "percent-encoding" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.10.1" @@ -1078,6 +1418,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1156,6 +1516,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettyplease" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fead41e178796ef8274dc612a7d8ce4c7e10ca35cd2c5b5ad24cac63aeb6c0" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1189,6 +1559,59 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e" +dependencies = [ + "bytes", + "prost", +] + [[package]] name = "quote" version = "1.0.21" @@ -1292,6 +1715,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "sendfd" version = "0.4.3" @@ -1440,6 +1869,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "tempfile" version = "3.3.0" @@ -1583,6 +2018,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "1.8.0" @@ -1594,6 +2039,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -1606,8 +2062,105 @@ dependencies = [ "futures-sink", "pin-project-lite", "tokio", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b9af819e54b8f33d453655bef9b9acc171568fb49523078d0cc4e7484200ec" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c6fd7c2581e36d63388a9e04c350c21beb7a8b059580b2e93993c526899ddc" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", ] +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -1615,6 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1641,6 +2195,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" version = "0.1.3" @@ -1652,6 +2216,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" version = "0.3.16" @@ -1666,6 +2244,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1797,6 +2381,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1866,6 +2460,17 @@ dependencies = [ "cc", ] +[[package]] +name = "which" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +dependencies = [ + "either", + "libc", + "once_cell", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cross.toml b/Cross.toml index 3b899547a5..7fa678a89f 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,2 +1,2 @@ [build] -pre-build = ["apt-get update && apt-get install -y capnproto"] +pre-build = ["apt-get update && apt-get install -y capnproto protobuf-compiler"] diff --git a/conmon-rs/server/Cargo.toml b/conmon-rs/server/Cargo.toml index 2b795e846d..a2f1544080 100644 --- a/conmon-rs/server/Cargo.toml +++ b/conmon-rs/server/Cargo.toml @@ -22,6 +22,8 @@ memchr = "2.5.0" multimap = "0.8.3" nix = "0.25.0" notify = "5.0.0" +opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } +opentelemetry-otlp = "0.11.0" prctl = "1.0.0" regex = "1.6.0" sendfd = { version = "0.4.3", features = ["tokio"] } @@ -34,12 +36,14 @@ tokio-eventfd = "0.2.0" tokio-fd = "0.3.0" tokio-util = { version = "0.7.4", features = ["compat"] } tracing = "0.1.37" +tracing-opentelemetry = "0.18.0" tracing-subscriber = "0.3.16" tz-rs = "0.6.14" uuid = { version = "1.1.2", features = ["v4", "fast-rng", "macro-diagnostics"] } [build-dependencies] shadow-rs = "=0.16.1" +dashmap = "=5.2.0" [dev-dependencies] mockall = "0.11.2" diff --git a/conmon-rs/server/src/config.rs b/conmon-rs/server/src/config.rs index b4511e4c4b..98f99edcfa 100644 --- a/conmon-rs/server/src/config.rs +++ b/conmon-rs/server/src/config.rs @@ -93,7 +93,7 @@ pub struct Config { long("skip-fork"), value_name("SKIP_FORK") )] - /// Do not fork if true + /// Do not fork if true. skip_fork: bool, #[get_copy = "pub"] @@ -105,8 +105,28 @@ pub struct Config { value_enum, value_name("MANAGER") )] - /// Select the cgroup manager to be used + /// Select the cgroup manager to be used. cgroup_manager: CgroupManager, + + #[get_copy = "pub"] + #[clap( + env(concat!(prefix!(), "ENABLE_TRACING")), + long("enable-tracing"), + short('e'), + )] + /// Enable OpenTelemetry tracing. + enable_tracing: bool, + + #[get = "pub"] + #[clap( + default_value("http://localhost:4317"), + env(concat!(prefix!(), "TRACING_ENDPOINT")), + long("tracing-endpoint"), + short('t'), + value_name("URL") + )] + /// OpenTelemetry GRPC endpoint to be used for tracing. + tracing_endpoint: String, } #[derive( diff --git a/conmon-rs/server/src/init.rs b/conmon-rs/server/src/init.rs index cd27069438..1566e7410c 100644 --- a/conmon-rs/server/src/init.rs +++ b/conmon-rs/server/src/init.rs @@ -6,7 +6,6 @@ use std::{ io::{self, ErrorKind, Write}, path::Path, }; -use tracing::info; #[cfg(test)] use mockall::{automock, predicate::*}; @@ -36,7 +35,7 @@ where { match err.kind() { ErrorKind::PermissionDenied => { - info!("Missing sufficient privileges to adjust OOM score") + println!("Missing sufficient privileges to adjust OOM score") } _ => bail!("adjusting OOM score {}", err), } diff --git a/conmon-rs/server/src/server.rs b/conmon-rs/server/src/server.rs index 0cba768839..9207bbc47a 100644 --- a/conmon-rs/server/src/server.rs +++ b/conmon-rs/server/src/server.rs @@ -12,6 +12,7 @@ use crate::{ use anyhow::{format_err, Context, Result}; use capnp::text_list::Reader; use capnp_rpc::{rpc_twoparty_capnp::Side, twoparty, RpcSystem}; +use clap::crate_name; use conmon_common::conmon_capnp::conmon; use futures::{AsyncReadExt, FutureExt}; use getset::Getters; @@ -21,7 +22,18 @@ use nix::{ sys::signal::Signal, unistd::{fork, ForkResult}, }; -use std::{fs::File, io::Write, path::Path, process, str::FromStr, sync::Arc}; +use opentelemetry::{ + global, + runtime::Tokio, + sdk::{ + propagation::TraceContextPropagator, + trace::{self, Tracer}, + Resource, + }, + KeyValue, +}; +use opentelemetry_otlp::{ExportConfig, Protocol, WithExportConfig}; +use std::{fs::File, io::Write, path::Path, process, str::FromStr, sync::Arc, time::Duration}; use tokio::{ fs, runtime::{Builder, Handle}, @@ -30,8 +42,11 @@ use tokio::{ task::{self, LocalSet}, }; use tokio_util::compat::TokioAsyncReadCompatExt; -use tracing::{debug, debug_span, info, Instrument}; -use tracing_subscriber::{filter::LevelFilter, prelude::*}; +use tracing::{debug, debug_span, info, Instrument, Subscriber}; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::{ + filter::LevelFilter, layer::SubscriberExt, prelude::*, registry::LookupSpan, +}; use twoparty::VatNetwork; #[derive(Debug, Getters)] @@ -59,7 +74,6 @@ impl Server { process::exit(0); } - server.init_logging().context("init logging")?; server.config().validate().context("validate config")?; Self::init().context("init self")?; @@ -107,7 +121,7 @@ impl Server { fn init_logging(&self) -> Result<()> { let level = LevelFilter::from_str(self.config().log_level().as_ref()) .context("convert log level filter")?; - let registry = tracing_subscriber::registry(); + let registry = tracing_subscriber::registry().with(self.telemetry_layer()?); match self.config().log_driver() { LogDriver::Stdout => { @@ -139,8 +153,39 @@ impl Server { Ok(()) } + /// Return the telemetry layer if tracing is enabled. + fn telemetry_layer(&self) -> Result>> + where + T: Subscriber + for<'span> LookupSpan<'span>, + { + if !self.config.enable_tracing() { + return Ok(None); + } + + global::set_text_map_propagator(TraceContextPropagator::new()); + let exporter = opentelemetry_otlp::new_exporter() + .tonic() + .with_export_config(ExportConfig { + endpoint: self.config.tracing_endpoint().into(), + timeout: Duration::from_secs(3), + protocol: Protocol::Grpc, + }); + let tracer = + opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + .with_trace_config(trace::config().with_resource(Resource::new(vec![ + KeyValue::new("service.name", crate_name!()), + ]))) + .install_batch(Tokio)?; + + Ok(tracing_opentelemetry::layer().with_tracer(tracer).into()) + } + /// Spwans all required tokio tasks. async fn spawn_tasks(self) -> Result<()> { + self.init_logging().context("init logging")?; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); let socket = self.config().socket(); let reaper = self.reaper.clone(); @@ -201,12 +246,16 @@ impl Server { async fn start_backend(self, mut shutdown_rx: oneshot::Receiver<()>) -> Result<()> { let listener = Listener::::default().bind_long_path(&self.config().socket())?; + let enable_tracing = self.config().enable_tracing(); let client: conmon::Client = capnp_rpc::new_client(self); loop { let stream = tokio::select! { _ = &mut shutdown_rx => { debug!("Received shutdown message"); + if enable_tracing { + global::shutdown_tracer_provider(); + } return Ok(()) } stream = listener.accept() => { diff --git a/contrib/tracing/docker-compose.yaml b/contrib/tracing/docker-compose.yaml new file mode 100644 index 0000000000..2f50608ec7 --- /dev/null +++ b/contrib/tracing/docker-compose.yaml @@ -0,0 +1,22 @@ +version: "2" +services: + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" + - "14268" + - "14250" + + otel-collector: + image: otel/opentelemetry-collector:latest + command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "1888:1888" + - "13133:13133" + - "4317:4317" + - "4318:4318" + - "55670:55679" + depends_on: + - jaeger-all-in-one diff --git a/contrib/tracing/otel-collector-config.yaml b/contrib/tracing/otel-collector-config.yaml new file mode 100644 index 0000000000..ca30173e85 --- /dev/null +++ b/contrib/tracing/otel-collector-config.yaml @@ -0,0 +1,36 @@ +receivers: + otlp: + protocols: + http: + grpc: + +exporters: + logging: + loglevel: debug + + jaeger: + endpoint: jaeger-all-in-one:14250 + tls: + insecure: true + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, jaeger] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [logging]