[BACKPORT 2024.1][#22842] docdb: Improve usability of stack trace tra…

…cking endpoints

Summary:
Original commit: b2e8ee7 / D36190
This diff makes the following changes to improve usability of the stack trace
tracking endpoints (/io-stack-traces, /debug-stack-traces):

1. Add a button to /utilz for /io-stack-traces:

{image uri=https://github.com/yugabyte/yugabyte-db/assets/6742815/c3db1e87-a0f2-4fb7-8aa2-bff65638220d , width=1000px}

2. Add a link to reset tracking (/reset-stack-traces).

3. Add the tracking period to the endpoints.

{image uri=https://github.com/yugabyte/yugabyte-db/assets/6742815/9de9f5f0-b63e-4618-bafb-cb89f01ea4ec}

4. Reset stack trace tracking when the track_stack_traces gflag is set.
Jira: DB-11739

Test Plan: Opened /io-stack-traces, /debug-stack-traces, /utilz, and tested toggling track_stack_traces.

Reviewers: asrivastava

Reviewed By: asrivastava

Subscribers: ybase

Tags: #jenkins-ready

Differential Revision: https://phorge.dev.yugabyte.com/D36212

src/yb/server/default-path-handlers.cc

@@ -665,7 +665,14 @@ static void HandleGetVersionInfo(
   jw.EndObject();
 }
 
-static void IOStackTraceHandler(const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
+template<typename WeightFormatter = std::identity>
+static void StackTraceTrackerHandler(
+    const Webserver::WebRequest& req,
+    Webserver::WebResponse* resp,
+    std::string_view title,
+    const std::unordered_map<StackTraceTrackingGroup, std::string>& groups,
+    std::string_view weight_header,
+    WeightFormatter format_weight = {}) {
   std::stringstream& output = resp->output;
 
   if (!GetAtomicFlag(&FLAGS_track_stack_traces)) {
@@ -680,62 +687,65 @@ static void IOStackTraceHandler(const Webserver::WebRequest& req, Webserver::Web
   std::sort(traces.begin(), traces.end(),
             [](const auto& left, const auto& right) { return left.weight > right.weight; });
 
-  output << tags.header << "I/O stack traces" << tags.end_header;
+  output << tags.header << title << tags.end_header;
 
-  auto stack_traces = html_print_helper.CreateTablePrinter(
-      "stack_traces", {"Type", "Count", "Bytes", "Stack Trace"});
+  auto tracking_start = GetLastStackTraceTrackerResetTime();
+  auto tracking_end = MonoTime::Now();
+  output << "Tracking Period: "
+         << tracking_start.ToFormattedString() << " to " << tracking_end.ToFormattedString()
+         << " (" << (tracking_end - tracking_start).ToString() << ")" << tags.line_break;
+  output << "<a href=\"/reset-stack-traces\">Reset Tracking</a>" << tags.line_break;
 
-  for (const auto& entry : traces) {
-    if (entry.count == 0 ||
-        (entry.group != StackTraceTrackingGroup::kReadIO &&
-         entry.group != StackTraceTrackingGroup::kWriteIO)) {
-      continue;
-    }
-    stack_traces.AddRow(
-        entry.group == StackTraceTrackingGroup::kReadIO ? "Read" : "Write",
-        entry.count,
-        HumanReadableNumBytes::ToString(entry.weight),
-        tags.pre_tag + EscapeForHtmlToString(entry.symbolized_trace) + tags.end_pre_tag);
+  std::vector<std::string> column_names;
+  if (groups.size() > 1) {
+    column_names.emplace_back("Type");
   }
+  column_names.emplace_back("Count");
+  column_names.emplace_back(weight_header);
+  column_names.emplace_back("Stack Trace");
 
-  stack_traces.Print();
-}
-
-static void DebugStackTraceHandler(const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
-  std::stringstream& output = resp->output;
-
-  if (!GetAtomicFlag(&FLAGS_track_stack_traces)) {
-    output << "track_stack_traces must be turned on to use this page.";
-    return;
-  }
-
-  Tags tags(false /* as_text */);
-  HtmlPrintHelper html_print_helper(output);
-
-  auto traces = GetTrackedStackTraces();
-  std::sort(traces.begin(), traces.end(),
-            [](const auto& left, const auto& right) { return left.count > right.count; });
-
-  output << tags.header << "Tracked stack traces" << tags.end_header;
-
-  auto stack_traces = html_print_helper.CreateTablePrinter(
-      "stack_traces", {"Count", "Stack Trace"});
+  auto stack_traces = html_print_helper.CreateTablePrinter("stack_traces", column_names);
 
   for (const auto& entry : traces) {
-    if (entry.count == 0 || entry.group != StackTraceTrackingGroup::kDebugging) {
+    auto group_itr = groups.find(entry.group);
+    if (entry.count == 0 || group_itr == groups.end()) {
       continue;
     }
-    stack_traces.AddRow(
-        entry.count,
-        tags.pre_tag + EscapeForHtmlToString(entry.symbolized_trace) + tags.end_pre_tag);
+
+    auto count = entry.count;
+    auto weight = format_weight(entry.weight);
+    auto stack = tags.pre_tag + EscapeForHtmlToString(entry.symbolized_trace) + tags.end_pre_tag;
+
+    if (groups.size() > 1) {
+      stack_traces.AddRow(group_itr->second, count, weight, stack);
+    } else {
+      stack_traces.AddRow(count, weight, stack);
+    }
   }
 
   stack_traces.Print();
 }
 
+static void IOStackTraceHandler(const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
+  StackTraceTrackerHandler(
+      req, resp, "I/O stack traces",
+      {{StackTraceTrackingGroup::kReadIO, "Read"}, {StackTraceTrackingGroup::kWriteIO, "Write"}},
+      "Bytes", &HumanReadableNumBytes::ToString);
+}
+
+static void DebugStackTraceHandler(const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
+  StackTraceTrackerHandler(
+      req, resp, "Tracked stack traces",
+      {{StackTraceTrackingGroup::kDebugging, "Debugging"}},
+      "Weight");
+}
+
 static void ResetStackTraceHandler(const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
   ResetTrackedStackTraces();
-  resp->output << "Tracked stack traces reset.";
+
+  Tags tags(false /* as_text */);
+  resp->output << "Tracked stack traces reset." << tags.line_break
+               << "<a href=\"javascript:window.location=document.referrer\">Back</a>";
 }
 
 } // anonymous namespace

src/yb/server/server_base.cc

@@ -660,6 +660,7 @@ void RpcAndWebServerBase::DisplayGeneralInfoIcons(std::stringstream* output) {
   DisplayIconTile(output, "fa-microchip", "Threads", "/threadz");
   // Drives.
   DisplayIconTile(output, "fa-hdd-o", "Drives", "/drives");
+  DisplayIconTile(output, "fa-pencil-square-o", "I/O Stack Traces", "/io-stack-traces");
   // TLS.
   DisplayIconTile(output, "fa-lock", "TLS", "/tls");
   DisplayIconTile(output, "fa-times", "xCluster", "/xcluster");

src/yb/util/stack_trace_tracker.cc

@@ -22,6 +22,7 @@
 #include "yb/util/atomic.h"
 #include "yb/util/debug.h"
 #include "yb/util/flags.h"
+#include "yb/util/flags/flags_callback.h"
 #include "yb/util/stack_trace.h"
 #include "yb/util/unique_lock.h"
 
@@ -35,6 +36,8 @@ class ThreadStackTraceTracker;
 
 class GlobalStackTraceTracker {
  public:
+  GlobalStackTraceTracker(): last_reset_(MonoTime::Now()) {}
+
   void RegisterThread(std::thread::id id, ThreadStackTraceTracker* tracker) EXCLUDES(mutex_) {
     std::lock_guard lock(mutex_);
     thread_trackers_[id] = tracker;
@@ -71,18 +74,32 @@ class GlobalStackTraceTracker {
     return tracked;
   }
 
+  MonoTime GetLastStackTraceTrackerResetTime() EXCLUDES(mutex_) {
+    std::lock_guard lock(mutex_);
+    return last_reset_;
+  }
+
   void ResetTrackedStackTraces() EXCLUDES(mutex_);
 
  private:
   void MergeLocalTracker(ThreadStackTraceTracker* tracker) REQUIRES(mutex_);
 
   std::mutex mutex_;
+  MonoTime last_reset_ GUARDED_BY(mutex_);
   UnorderedStringMap<StackTraceEntry> traces_ GUARDED_BY(mutex_);
   std::unordered_map<std::thread::id, ThreadStackTraceTracker*> thread_trackers_ GUARDED_BY(mutex_);
 };
 
 static GlobalStackTraceTracker global_tracker;
 
+void TrackStackTraceToggleCallback() {
+  if (GetAtomicFlag(&FLAGS_track_stack_traces)) {
+    global_tracker.ResetTrackedStackTraces();
+  }
+}
+
+REGISTER_CALLBACK(track_stack_traces, "Stack Trace Toggle Callback", TrackStackTraceToggleCallback);
+
 // This class is thread local and accessed by one thread only, except for when we
 // MergeToGlobal/Reset.
 class ThreadStackTraceTracker {
@@ -149,6 +166,8 @@ void GlobalStackTraceTracker::ResetTrackedStackTraces() {
     entry.count = 0;
     entry.weight = 0;
   }
+
+  last_reset_ = MonoTime::Now();
 }
 
 void GlobalStackTraceTracker::MergeLocalTracker(ThreadStackTraceTracker* tracker) {
@@ -180,6 +199,10 @@ void ResetTrackedStackTraces() {
   global_tracker.ResetTrackedStackTraces();
 }
 
+MonoTime GetLastStackTraceTrackerResetTime() {
+  return global_tracker.GetLastStackTraceTrackerResetTime();
+}
+
 std::vector<StackTraceEntry> GetTrackedStackTraces() {
   return global_tracker.GetTrackedStackTraces();
 }

src/yb/util/stack_trace_tracker.h

@@ -36,6 +36,8 @@ void TrackStackTrace(
 
 void ResetTrackedStackTraces();
 
+MonoTime GetLastStackTraceTrackerResetTime();
+
 std::vector<StackTraceEntry> GetTrackedStackTraces();
 
 void DumpTrackedStackTracesToLog(StackTraceTrackingGroup group);