refactor bpf histogram indexing

src/common/bpf/histogram.h

@@ -1,16 +1,16 @@
 // Helpers for converting values to histogram indices. 
 
+#define HISTOGRAM_BUCKETS_POW_4 976
+#define HISTOGRAM_BUCKETS_POW_5 1920
+#define HISTOGRAM_BUCKETS_POW_6 3776
+#define HISTOGRAM_BUCKETS_POW_7 7424
+
 // Function to count leading zeros, since we cannot use the builtin CLZ from
 // within BPF. But since we also can't loop, this is implemented as a binary
-// search with a maximum of 6 branches.
+// search with a maximum of 6 branches. 
 static u32 clz(u64 value) {
     u32 count = 0;
 
-    // quick return if value is 0
-    if (!value) {
-        return 64;
-    }
-
     // binary search to find number of leading zeros
     if (value & 0xFFFFFFFF00000000) {
         if (value & 0xFFFF000000000000) {
@@ -203,25 +203,23 @@ static u32 clz(u64 value) {
     } else {
         return 63;
     }
+
+    return 64;
 }
 
 // base-2 histogram indexing function that is compatible with Rust `histogram`
-// crate for m = 0, r = 8, n = 64 this gives us the ability to store counts for
-// values from 1 -> u64::MAX and uses 7424 buckets per histogram, which occupies
-// 58KB of space in kernelspace (where we use 64bit counters)
-static u32 value_to_index(u64 value) {
-    if (value == 0) {
-        return 0;
-    }
-
-    u64 h = 63 - clz(value);
-    // h < r
-    if (h < 8) {
+// crate.
+//
+// See the indexing logic here:
+// https://github.com/pelikan-io/rustcommon/blob/main/histogram/src/config.rs
+static u32 value_to_index(u64 value, u8 grouping_power) {
+    if (value < (2 << grouping_power)) {
         return value;
     } else {
-        // d = h - r + 1
-        u64 d = h - 7;
-        // ((d + 1) * G + ((value - (1 << h)) >> (m + d)))
-        return ((d + 1) * 128) + ((value - (1 << h)) >> d);
+        u64 power = 63 - clz(value);
+        u64 bin = power - grouping_power + 1;
+        u64 offset = (value - (1 << power)) >> (power - grouping_power);
+
+        return (bin * (1 << grouping_power) + offset);
     }
 }

src/samplers/block_io/linux/latency/mod.bpf.c

@@ -11,6 +11,7 @@
 extern int LINUX_KERNEL_VERSION __kconfig;
 
 #define COUNTER_GROUP_WIDTH 8
+#define HISTOGRAM_POWER 7
 #define MAX_CPUS 1024
 
 #define REQ_OP_BITS	8
@@ -106,7 +107,7 @@ static int handle_block_rq_complete(struct request *rq, int error, unsigned int
 		}
     }
 
-	idx = value_to_index(nr_bytes);
+	idx = value_to_index(nr_bytes, HISTOGRAM_POWER);
 	cnt = bpf_map_lookup_elem(&size, &idx);
 
 	if (cnt) {
@@ -121,7 +122,7 @@ static int handle_block_rq_complete(struct request *rq, int error, unsigned int
 	if (*tsp <= ts) {
 		delta = ts - *tsp;
 
-		idx = value_to_index(delta);
+		idx = value_to_index(delta, HISTOGRAM_POWER);
 		cnt = bpf_map_lookup_elem(&latency, &idx);
 
 		if (cnt) {

src/samplers/block_io/linux/latency/mod.rs

@@ -58,6 +58,19 @@ impl Biolat {
             .load()
             .map_err(|e| error!("failed to load bpf program: {e}"))?;
 
+        debug!(
+            "{NAME} block_rq_insert() BPF instruction count: {}",
+            skel.progs().block_rq_insert().insn_cnt()
+        );
+        debug!(
+            "{NAME} block_rq_issue() BPF instruction count: {}",
+            skel.progs().block_rq_issue().insn_cnt()
+        );
+        debug!(
+            "{NAME} block_rq_complete() BPF instruction count: {}",
+            skel.progs().block_rq_complete().insn_cnt()
+        );
+
         skel.attach()
             .map_err(|e| error!("failed to attach bpf program: {e}"))?;
 

src/samplers/cpu/linux/usage/bpf.rs

@@ -56,6 +56,11 @@ impl CpuUsage {
             .load()
             .map_err(|e| error!("failed to load bpf program: {e}"))?;
 
+        debug!(
+            "{NAME} cpuacct_account_field() BPF instruction count: {}",
+            skel.progs().cpuacct_account_field_kprobe().insn_cnt()
+        );
+
         skel.attach()
             .map_err(|e| error!("failed to attach bpf program: {e}"))?;
 

src/samplers/network/linux/traffic/bpf.rs

@@ -50,6 +50,15 @@ impl NetworkTraffic {
             .load()
             .map_err(|e| error!("failed to load bpf program: {e}"))?;
 
+        debug!(
+            "{NAME} netif_receive_skb() BPF instruction count: {}",
+            skel.progs().netif_receive_skb().insn_cnt()
+        );
+        debug!(
+            "{NAME} tcp_cleanup_rbuf() BPF instruction count: {}",
+            skel.progs().tcp_cleanup_rbuf().insn_cnt()
+        );
+
         skel.attach()
             .map_err(|e| error!("failed to attach bpf program: {e}"))?;
 

src/samplers/scheduler/linux/runqueue/mod.bpf.c

@@ -15,7 +15,7 @@
 #include <bpf/bpf_helpers.h>
 
 #define COUNTER_GROUP_WIDTH 8
-#define HISTOGRAM_BUCKETS 7424
+#define HISTOGRAM_POWER 7
 #define MAX_CPUS 1024
 #define MAX_PID 4194304
 
@@ -90,23 +90,23 @@ struct {
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } runqlat SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } running SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } offcpu SEC(".maps");
 
 /* record enqueue timestamp */
@@ -185,7 +185,7 @@ int handle__sched_switch(u64 *ctx)
 			delta_ns = ts - *tsp;
 
 			// update histogram
-			idx = value_to_index(delta_ns);
+			idx = value_to_index(delta_ns, HISTOGRAM_POWER);
 			cnt = bpf_map_lookup_elem(&running, &idx);
 			if (cnt) {
 				__sync_fetch_and_add(cnt, 1);
@@ -215,7 +215,7 @@ int handle__sched_switch(u64 *ctx)
 		delta_ns = ts - *tsp;
 
 		// update the histogram
-		idx = value_to_index(delta_ns);
+		idx = value_to_index(delta_ns, HISTOGRAM_POWER);
 		cnt = bpf_map_lookup_elem(&runqlat, &idx);
 		if (cnt) {
 			__sync_fetch_and_add(cnt, 1);
@@ -233,7 +233,7 @@ int handle__sched_switch(u64 *ctx)
 				offcpu_ns = offcpu_ns - delta_ns;
 
 				// update the histogram
-				idx = value_to_index(offcpu_ns);
+				idx = value_to_index(offcpu_ns, HISTOGRAM_POWER);
 				cnt = bpf_map_lookup_elem(&offcpu, &idx);
 				if (cnt) {
 					__sync_fetch_and_add(cnt, 1);

src/samplers/scheduler/linux/runqueue/mod.rs

@@ -60,6 +60,19 @@ impl Runqlat {
             .load()
             .map_err(|e| error!("failed to load bpf program: {e}"))?;
 
+        debug!(
+            "{NAME} handle__sched_wakeup() BPF instruction count: {}",
+            skel.progs().handle__sched_wakeup().insn_cnt()
+        );
+        debug!(
+            "{NAME} handle__sched_wakeup_new() BPF instruction count: {}",
+            skel.progs().handle__sched_wakeup_new().insn_cnt()
+        );
+        debug!(
+            "{NAME} handle__sched_switch() BPF instruction count: {}",
+            skel.progs().handle__sched_switch().insn_cnt()
+        );
+
         skel.attach()
             .map_err(|e| error!("failed to attach bpf program: {e}"))?;
 

src/samplers/syscall/linux/latency/mod.bpf.c

@@ -18,7 +18,7 @@
 #include <bpf/bpf_core_read.h>
 
 #define COUNTER_GROUP_WIDTH 8
-#define HISTOGRAM_BUCKETS 7424
+#define HISTOGRAM_POWER 7
 #define MAX_CPUS 1024
 #define MAX_SYSCALL_ID 1024
 #define MAX_PID 4194304
@@ -48,63 +48,63 @@ struct {
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } total_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } read_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } write_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } poll_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } lock_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } time_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } sleep_latency SEC(".maps");
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, HISTOGRAM_BUCKETS);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } socket_latency SEC(".maps");
 
 // provides a lookup table from syscall id to a counter index offset
@@ -188,7 +188,7 @@ int sys_exit(struct trace_event_raw_sys_exit *args)
 	*start_ts = 0;
 
 	// calculate the histogram index for this latency value
-	idx = value_to_index(lat);
+	idx = value_to_index(lat, HISTOGRAM_POWER);
 
 	// update the total latency histogram
 	cnt = bpf_map_lookup_elem(&total_latency, &idx);

src/samplers/syscall/linux/latency/mod.rs

@@ -59,6 +59,15 @@ impl Syscall {
             .load()
             .map_err(|e| error!("failed to load bpf program: {e}"))?;
 
+        debug!(
+            "{NAME} sys_enter() BPF instruction count: {}",
+            skel.progs().sys_enter().insn_cnt()
+        );
+        debug!(
+            "{NAME} sys_exit() BPF instruction count: {}",
+            skel.progs().sys_exit().insn_cnt()
+        );
+
         skel.attach()
             .map_err(|e| error!("failed to attach bpf program: {e}"))?;
 

src/samplers/tcp/linux/packet_latency/mod.bpf.c

@@ -16,6 +16,8 @@
 #include <bpf/bpf_core_read.h>
 #include <bpf/bpf_tracing.h>
 
+#define HISTOGRAM_POWER 7
+
 #define MAX_ENTRIES	10240
 #define AF_INET		2
 #define NO_EXIST    1
@@ -32,7 +34,7 @@ struct {
 	__uint(map_flags, BPF_F_MMAPABLE);
 	__type(key, u32);
 	__type(value, u64);
-	__uint(max_entries, 7424);
+	__uint(max_entries, HISTOGRAM_BUCKETS_POW_7);
 } latency SEC(".maps");
 
 static __always_inline __u64 get_sock_ident(struct sock *sk)
@@ -83,7 +85,7 @@ static int handle_tcp_rcv_space_adjust(void *ctx, struct sock *sk)
 
 	delta_ns = (now - *tsp);
 
-	idx = value_to_index(delta_ns);
+	idx = value_to_index(delta_ns, HISTOGRAM_POWER);
 	cnt = bpf_map_lookup_elem(&latency, &idx);
 
 	if (cnt) {

src/samplers/tcp/linux/packet_latency/mod.rs

@@ -57,6 +57,19 @@ impl PacketLatency {
             .load()
             .map_err(|e| error!("failed to load bpf program: {e}"))?;
 
+        debug!(
+            "{NAME} tcp_probe() BPF instruction count: {}",
+            skel.progs().tcp_probe().insn_cnt()
+        );
+        debug!(
+            "{NAME} tcp_rcv_space_adjust() BPF instruction count: {}",
+            skel.progs().tcp_rcv_space_adjust().insn_cnt()
+        );
+        debug!(
+            "{NAME} tcp_destroy_sock() BPF instruction count: {}",
+            skel.progs().tcp_destroy_sock().insn_cnt()
+        );
+
         skel.attach()
             .map_err(|e| error!("failed to attach bpf program: {e}"))?;