diff --git a/src/common/bpf/histogram.h b/src/common/bpf/histogram.h index 3aaafcb5..eba4bb42 100644 --- a/src/common/bpf/histogram.h +++ b/src/common/bpf/histogram.h @@ -1,16 +1,16 @@ // Helpers for converting values to histogram indices. +#define HISTOGRAM_BUCKETS_POW_4 976 +#define HISTOGRAM_BUCKETS_POW_5 1920 +#define HISTOGRAM_BUCKETS_POW_6 3776 +#define HISTOGRAM_BUCKETS_POW_7 7424 + // Function to count leading zeros, since we cannot use the builtin CLZ from // within BPF. But since we also can't loop, this is implemented as a binary -// search with a maximum of 6 branches. +// search with a maximum of 6 branches. static u32 clz(u64 value) { u32 count = 0; - // quick return if value is 0 - if (!value) { - return 64; - } - // binary search to find number of leading zeros if (value & 0xFFFFFFFF00000000) { if (value & 0xFFFF000000000000) { @@ -203,25 +203,23 @@ static u32 clz(u64 value) { } else { return 63; } + + return 64; } // base-2 histogram indexing function that is compatible with Rust `histogram` -// crate for m = 0, r = 8, n = 64 this gives us the ability to store counts for -// values from 1 -> u64::MAX and uses 7424 buckets per histogram, which occupies -// 58KB of space in kernelspace (where we use 64bit counters) -static u32 value_to_index(u64 value) { - if (value == 0) { - return 0; - } - - u64 h = 63 - clz(value); - // h < r - if (h < 8) { +// crate. +// +// See the indexing logic here: +// https://github.com/pelikan-io/rustcommon/blob/main/histogram/src/config.rs +static u32 value_to_index(u64 value, u8 grouping_power) { + if (value < (2 << grouping_power)) { return value; } else { - // d = h - r + 1 - u64 d = h - 7; - // ((d + 1) * G + ((value - (1 << h)) >> (m + d))) - return ((d + 1) * 128) + ((value - (1 << h)) >> d); + u64 power = 63 - clz(value); + u64 bin = power - grouping_power + 1; + u64 offset = (value - (1 << power)) >> (power - grouping_power); + + return (bin * (1 << grouping_power) + offset); } } diff --git a/src/samplers/block_io/linux/latency/mod.bpf.c b/src/samplers/block_io/linux/latency/mod.bpf.c index e84e7d3d..c9281a9c 100644 --- a/src/samplers/block_io/linux/latency/mod.bpf.c +++ b/src/samplers/block_io/linux/latency/mod.bpf.c @@ -11,6 +11,7 @@ extern int LINUX_KERNEL_VERSION __kconfig; #define COUNTER_GROUP_WIDTH 8 +#define HISTOGRAM_POWER 7 #define MAX_CPUS 1024 #define REQ_OP_BITS 8 @@ -106,7 +107,7 @@ static int handle_block_rq_complete(struct request *rq, int error, unsigned int } } - idx = value_to_index(nr_bytes); + idx = value_to_index(nr_bytes, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&size, &idx); if (cnt) { @@ -121,7 +122,7 @@ static int handle_block_rq_complete(struct request *rq, int error, unsigned int if (*tsp <= ts) { delta = ts - *tsp; - idx = value_to_index(delta); + idx = value_to_index(delta, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&latency, &idx); if (cnt) { diff --git a/src/samplers/block_io/linux/latency/mod.rs b/src/samplers/block_io/linux/latency/mod.rs index 4237d059..f2b3ddb3 100644 --- a/src/samplers/block_io/linux/latency/mod.rs +++ b/src/samplers/block_io/linux/latency/mod.rs @@ -58,6 +58,19 @@ impl Biolat { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} block_rq_insert() BPF instruction count: {}", + skel.progs().block_rq_insert().insn_cnt() + ); + debug!( + "{NAME} block_rq_issue() BPF instruction count: {}", + skel.progs().block_rq_issue().insn_cnt() + ); + debug!( + "{NAME} block_rq_complete() BPF instruction count: {}", + skel.progs().block_rq_complete().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/cpu/linux/usage/bpf.rs b/src/samplers/cpu/linux/usage/bpf.rs index 80dbe66d..5885c30d 100644 --- a/src/samplers/cpu/linux/usage/bpf.rs +++ b/src/samplers/cpu/linux/usage/bpf.rs @@ -56,6 +56,11 @@ impl CpuUsage { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} cpuacct_account_field() BPF instruction count: {}", + skel.progs().cpuacct_account_field_kprobe().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/network/linux/traffic/bpf.rs b/src/samplers/network/linux/traffic/bpf.rs index b1761cb3..47d781e9 100644 --- a/src/samplers/network/linux/traffic/bpf.rs +++ b/src/samplers/network/linux/traffic/bpf.rs @@ -50,6 +50,15 @@ impl NetworkTraffic { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} netif_receive_skb() BPF instruction count: {}", + skel.progs().netif_receive_skb().insn_cnt() + ); + debug!( + "{NAME} tcp_cleanup_rbuf() BPF instruction count: {}", + skel.progs().tcp_cleanup_rbuf().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/scheduler/linux/runqueue/mod.bpf.c b/src/samplers/scheduler/linux/runqueue/mod.bpf.c index d342ab3c..4c0f5062 100644 --- a/src/samplers/scheduler/linux/runqueue/mod.bpf.c +++ b/src/samplers/scheduler/linux/runqueue/mod.bpf.c @@ -15,7 +15,7 @@ #include #define COUNTER_GROUP_WIDTH 8 -#define HISTOGRAM_BUCKETS 7424 +#define HISTOGRAM_POWER 7 #define MAX_CPUS 1024 #define MAX_PID 4194304 @@ -90,7 +90,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } runqlat SEC(".maps"); struct { @@ -98,7 +98,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } running SEC(".maps"); struct { @@ -106,7 +106,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } offcpu SEC(".maps"); /* record enqueue timestamp */ @@ -185,7 +185,7 @@ int handle__sched_switch(u64 *ctx) delta_ns = ts - *tsp; // update histogram - idx = value_to_index(delta_ns); + idx = value_to_index(delta_ns, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&running, &idx); if (cnt) { __sync_fetch_and_add(cnt, 1); @@ -215,7 +215,7 @@ int handle__sched_switch(u64 *ctx) delta_ns = ts - *tsp; // update the histogram - idx = value_to_index(delta_ns); + idx = value_to_index(delta_ns, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&runqlat, &idx); if (cnt) { __sync_fetch_and_add(cnt, 1); @@ -233,7 +233,7 @@ int handle__sched_switch(u64 *ctx) offcpu_ns = offcpu_ns - delta_ns; // update the histogram - idx = value_to_index(offcpu_ns); + idx = value_to_index(offcpu_ns, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&offcpu, &idx); if (cnt) { __sync_fetch_and_add(cnt, 1); diff --git a/src/samplers/scheduler/linux/runqueue/mod.rs b/src/samplers/scheduler/linux/runqueue/mod.rs index be8f8154..609133c2 100644 --- a/src/samplers/scheduler/linux/runqueue/mod.rs +++ b/src/samplers/scheduler/linux/runqueue/mod.rs @@ -60,6 +60,19 @@ impl Runqlat { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} handle__sched_wakeup() BPF instruction count: {}", + skel.progs().handle__sched_wakeup().insn_cnt() + ); + debug!( + "{NAME} handle__sched_wakeup_new() BPF instruction count: {}", + skel.progs().handle__sched_wakeup_new().insn_cnt() + ); + debug!( + "{NAME} handle__sched_switch() BPF instruction count: {}", + skel.progs().handle__sched_switch().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/syscall/linux/latency/mod.bpf.c b/src/samplers/syscall/linux/latency/mod.bpf.c index 3c1e1452..1d0ead3a 100644 --- a/src/samplers/syscall/linux/latency/mod.bpf.c +++ b/src/samplers/syscall/linux/latency/mod.bpf.c @@ -18,7 +18,7 @@ #include #define COUNTER_GROUP_WIDTH 8 -#define HISTOGRAM_BUCKETS 7424 +#define HISTOGRAM_POWER 7 #define MAX_CPUS 1024 #define MAX_SYSCALL_ID 1024 #define MAX_PID 4194304 @@ -48,7 +48,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } total_latency SEC(".maps"); struct { @@ -56,7 +56,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } read_latency SEC(".maps"); struct { @@ -64,7 +64,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } write_latency SEC(".maps"); struct { @@ -72,7 +72,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } poll_latency SEC(".maps"); struct { @@ -80,7 +80,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } lock_latency SEC(".maps"); struct { @@ -88,7 +88,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } time_latency SEC(".maps"); struct { @@ -96,7 +96,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } sleep_latency SEC(".maps"); struct { @@ -104,7 +104,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, HISTOGRAM_BUCKETS); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } socket_latency SEC(".maps"); // provides a lookup table from syscall id to a counter index offset @@ -188,7 +188,7 @@ int sys_exit(struct trace_event_raw_sys_exit *args) *start_ts = 0; // calculate the histogram index for this latency value - idx = value_to_index(lat); + idx = value_to_index(lat, HISTOGRAM_POWER); // update the total latency histogram cnt = bpf_map_lookup_elem(&total_latency, &idx); diff --git a/src/samplers/syscall/linux/latency/mod.rs b/src/samplers/syscall/linux/latency/mod.rs index ca1a686f..d58ec19b 100644 --- a/src/samplers/syscall/linux/latency/mod.rs +++ b/src/samplers/syscall/linux/latency/mod.rs @@ -59,6 +59,15 @@ impl Syscall { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} sys_enter() BPF instruction count: {}", + skel.progs().sys_enter().insn_cnt() + ); + debug!( + "{NAME} sys_exit() BPF instruction count: {}", + skel.progs().sys_exit().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/tcp/linux/packet_latency/mod.bpf.c b/src/samplers/tcp/linux/packet_latency/mod.bpf.c index 8f15893f..a7ed9062 100644 --- a/src/samplers/tcp/linux/packet_latency/mod.bpf.c +++ b/src/samplers/tcp/linux/packet_latency/mod.bpf.c @@ -16,6 +16,8 @@ #include #include +#define HISTOGRAM_POWER 7 + #define MAX_ENTRIES 10240 #define AF_INET 2 #define NO_EXIST 1 @@ -32,7 +34,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, 7424); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } latency SEC(".maps"); static __always_inline __u64 get_sock_ident(struct sock *sk) @@ -83,7 +85,7 @@ static int handle_tcp_rcv_space_adjust(void *ctx, struct sock *sk) delta_ns = (now - *tsp); - idx = value_to_index(delta_ns); + idx = value_to_index(delta_ns, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&latency, &idx); if (cnt) { diff --git a/src/samplers/tcp/linux/packet_latency/mod.rs b/src/samplers/tcp/linux/packet_latency/mod.rs index 944a5767..c6dc7214 100644 --- a/src/samplers/tcp/linux/packet_latency/mod.rs +++ b/src/samplers/tcp/linux/packet_latency/mod.rs @@ -57,6 +57,19 @@ impl PacketLatency { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} tcp_probe() BPF instruction count: {}", + skel.progs().tcp_probe().insn_cnt() + ); + debug!( + "{NAME} tcp_rcv_space_adjust() BPF instruction count: {}", + skel.progs().tcp_rcv_space_adjust().insn_cnt() + ); + debug!( + "{NAME} tcp_destroy_sock() BPF instruction count: {}", + skel.progs().tcp_destroy_sock().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/tcp/linux/receive/mod.bpf.c b/src/samplers/tcp/linux/receive/mod.bpf.c index d4f24f7c..6bc9356b 100644 --- a/src/samplers/tcp/linux/receive/mod.bpf.c +++ b/src/samplers/tcp/linux/receive/mod.bpf.c @@ -16,12 +16,14 @@ #include #include +#define HISTOGRAM_POWER 7 + struct { __uint(type, BPF_MAP_TYPE_ARRAY); __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, 7424); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } jitter SEC(".maps"); struct { @@ -29,7 +31,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, 7424); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } srtt SEC(".maps"); SEC("kprobe/tcp_rcv_established") @@ -49,7 +51,7 @@ int BPF_KPROBE(tcp_rcv_kprobe, struct sock *sk) // record nanoseconds. srtt_ns = 1000 * (u64) srtt_us >> 3; - idx = value_to_index(srtt_ns); + idx = value_to_index(srtt_ns, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&srtt, &idx); if (cnt) { @@ -60,7 +62,7 @@ int BPF_KPROBE(tcp_rcv_kprobe, struct sock *sk) // record nanoseconds. mdev_ns = 1000 * (u64) mdev_us >> 2; - idx = value_to_index(mdev_ns); + idx = value_to_index(mdev_ns, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&jitter, &idx); if (cnt) { diff --git a/src/samplers/tcp/linux/receive/mod.rs b/src/samplers/tcp/linux/receive/mod.rs index 53b2cc7a..59584a74 100644 --- a/src/samplers/tcp/linux/receive/mod.rs +++ b/src/samplers/tcp/linux/receive/mod.rs @@ -56,6 +56,11 @@ impl Receive { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} tcp_rcv() BPF instruction count: {}", + skel.progs().tcp_rcv_kprobe().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/tcp/linux/retransmit/mod.rs b/src/samplers/tcp/linux/retransmit/mod.rs index 39217122..5dc2fc29 100644 --- a/src/samplers/tcp/linux/retransmit/mod.rs +++ b/src/samplers/tcp/linux/retransmit/mod.rs @@ -55,6 +55,11 @@ impl Retransmit { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} tcp_retransmit_skb() BPF instruction count: {}", + skel.progs().tcp_retransmit_skb().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/tcp/linux/traffic/bpf.rs b/src/samplers/tcp/linux/traffic/bpf.rs index 526f9b7b..d7e59702 100644 --- a/src/samplers/tcp/linux/traffic/bpf.rs +++ b/src/samplers/tcp/linux/traffic/bpf.rs @@ -52,6 +52,15 @@ impl TcpTraffic { .load() .map_err(|e| error!("failed to load bpf program: {e}"))?; + debug!( + "{NAME} tcp_sendmsg() BPF instruction count: {}", + skel.progs().tcp_sendmsg().insn_cnt() + ); + debug!( + "{NAME} tcp_cleanup_rbuf() BPF instruction count: {}", + skel.progs().tcp_cleanup_rbuf().insn_cnt() + ); + skel.attach() .map_err(|e| error!("failed to attach bpf program: {e}"))?; diff --git a/src/samplers/tcp/linux/traffic/mod.bpf.c b/src/samplers/tcp/linux/traffic/mod.bpf.c index db81e312..8bcd1f1c 100644 --- a/src/samplers/tcp/linux/traffic/mod.bpf.c +++ b/src/samplers/tcp/linux/traffic/mod.bpf.c @@ -16,6 +16,8 @@ #include #include +#define HISTOGRAM_POWER 7 + /* Taken from kernel include/linux/socket.h. */ #define AF_INET 2 /* Internet IP Protocol */ #define AF_INET6 10 /* IP version 6 */ @@ -39,7 +41,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, 7424); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } rx_size SEC(".maps"); struct { @@ -47,7 +49,7 @@ struct { __uint(map_flags, BPF_F_MMAPABLE); __type(key, u32); __type(value, u64); - __uint(max_entries, 7424); + __uint(max_entries, HISTOGRAM_BUCKETS_POW_7); } tx_size SEC(".maps"); static int probe_ip(bool receiving, struct sock *sk, size_t size) @@ -72,7 +74,7 @@ static int probe_ip(bool receiving, struct sock *sk, size_t size) __sync_fetch_and_add(cnt, (u64) size); } - idx = value_to_index((u64) size); + idx = value_to_index((u64) size, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&rx_size, &idx); if (cnt) { @@ -93,7 +95,7 @@ static int probe_ip(bool receiving, struct sock *sk, size_t size) __sync_fetch_and_add(cnt, (u64) size); } - idx = value_to_index((u64) size); + idx = value_to_index((u64) size, HISTOGRAM_POWER); cnt = bpf_map_lookup_elem(&tx_size, &idx); if (cnt) {