Skip to content

Commit

Permalink
tracing: add /debug/tracez rendering the active spans
Browse files Browse the repository at this point in the history
/debug/tracez lets users take a snapshot of the active spans registry
and render the new snapshot, or one of the previously taken snapshots.
The Tracer can hold up to 10 snapshots in memory.

The PR description has a screenshot.

When visualizing a snapshot, the page lets you do a number of things:
1. List all the spans.
2. See the (current) stack trace for each span's goroutine (if the
   goroutine was still running at the time when the snapshot was
   captured). Stack traces can be toggled visible/hidden.
3. Sort the spans by name or start time.
4. Filter the span according to text search. The search works across
   the name and stack trace.
5. See the full trace that a particular span is part of.

For the table Javascript providing sorting and filtering, this patch
embeds the library from https://listjs.com/ .

Limitations:
- for now, only the registry of the local node is snapshotted. In the
  fuiture I'll collect info from all nodes.
- for now, the relationships between different spans are not represented
  in any way. I'll work on the ability to go from a span to the whole
  trace that the span is part of.
- for now, tags and structured and unstructured log messages that a span
  might have are not displayed in any way.

At the moment, span creation is not enabled in production by default
(i.e. the Tracer is put in TracingModeOnDemand by default, instead of
the required TracingModeActiveSpansRegistry). This patch does not change
that, so in order to benefit from /debug/tracez in all its glory, one
has to run with COCKROACH_REAL_SPANS=1 for now. Not for long, though.

Release note: None
  • Loading branch information
andreimatei committed Jan 20, 2022
1 parent 168c3e3 commit f87695e
Show file tree
Hide file tree
Showing 22 changed files with 787 additions and 25 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -1442,7 +1442,7 @@ ui-watch ui-watch-secure: $(UI_CCL_DLLS) pkg/ui/yarn.opt.installed
ui-clean: ## Remove build artifacts.
find pkg/ui/distccl/assets pkg/ui/distoss/assets -mindepth 1 -not -name .gitkeep -delete
rm -rf pkg/ui/assets.ccl.installed pkg/ui/assets.oss.installed
rm -rf pkg/ui/dist/*
rm -rf pkg/ui/dist_vendor/*
rm -f $(UI_PROTOS_CCL) $(UI_PROTOS_OSS)
rm -f pkg/ui/workspaces/db-console/*manifest.json
rm -rf pkg/ui/workspaces/cluster-ui/dist
Expand Down
2 changes: 2 additions & 0 deletions pkg/server/debug/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ go_library(
"//pkg/util/log/severity",
"//pkg/util/stop",
"//pkg/util/timeutil",
"//pkg/util/tracing",
"//pkg/util/tracing/tracingui",
"//pkg/util/uint128",
"@com_github_cockroachdb_errors//:errors",
"@com_github_cockroachdb_pebble//tool",
Expand Down
27 changes: 20 additions & 7 deletions pkg/server/debug/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import (
"github.com/cockroachdb/cockroach/pkg/storage"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/stop"
"github.com/cockroachdb/cockroach/pkg/util/tracing"
"github.com/cockroachdb/cockroach/pkg/util/tracing/tracingui"
"github.com/cockroachdb/errors"
pebbletool "github.com/cockroachdb/pebble/tool"
metrics "github.com/rcrowley/go-metrics"
Expand Down Expand Up @@ -61,14 +63,18 @@ var _ = func() *settings.StringSetting {

// Server serves the /debug/* family of tools.
type Server struct {
st *cluster.Settings
mux *http.ServeMux
spy logSpy
ambientCtx log.AmbientContext
st *cluster.Settings
mux *http.ServeMux
spy logSpy
}

// NewServer sets up a debug server.
func NewServer(
st *cluster.Settings, hbaConfDebugFn http.HandlerFunc, profiler pprofui.Profiler,
ambientContext log.AmbientContext,
st *cluster.Settings,
hbaConfDebugFn http.HandlerFunc,
profiler pprofui.Profiler,
) *Server {
mux := http.NewServeMux()

Expand Down Expand Up @@ -137,9 +143,10 @@ func NewServer(
})

return &Server{
st: st,
mux: mux,
spy: spy,
ambientCtx: ambientContext,
st: st,
mux: mux,
spy: spy,
}
}

Expand Down Expand Up @@ -232,6 +239,12 @@ func (ds *Server) RegisterClosedTimestampSideTransport(
})
}

// RegisterTracez registers the /debug/tracez handler, which renders snapshots
// of active spans.
func (ds *Server) RegisterTracez(tr *tracing.Tracer) {
tracingui.RegisterHTTPHandlers(ds.ambientCtx, ds.mux, tr)
}

// ServeHTTP serves various tools under the /debug endpoint.
func (ds *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
handler, _ := ds.mux.Handler(r)
Expand Down
3 changes: 2 additions & 1 deletion pkg/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -834,7 +834,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
}
sStatus.setStmtDiagnosticsRequester(sqlServer.execCfg.StmtDiagnosticsRecorder)
sStatus.baseStatusServer.sqlServer = sqlServer
debugServer := debug.NewServer(st, sqlServer.pgServer.HBADebugFn(), sStatus)
debugServer := debug.NewServer(cfg.BaseConfig.AmbientCtx, st, sqlServer.pgServer.HBADebugFn(), sStatus)
node.InitLogger(sqlServer.execCfg)

*lateBoundServer = Server{
Expand Down Expand Up @@ -1958,6 +1958,7 @@ func (s *Server) PreStart(ctx context.Context) error {
return errors.Wrapf(err, "failed to register engines with debug server")
}
s.debug.RegisterClosedTimestampSideTransport(s.ctSender, s.node.storeCfg.ClosedTimestampReceiver)
s.debug.RegisterTracez(s.cfg.Tracer)

s.ctSender.Run(ctx, state.nodeID)

Expand Down
2 changes: 1 addition & 1 deletion pkg/server/tenant.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ func StartTenant(
)

mux := http.NewServeMux()
debugServer := debug.NewServer(args.Settings, s.pgServer.HBADebugFn(), s.execCfg.SQLStatusServer)
debugServer := debug.NewServer(baseCfg.AmbientCtx, args.Settings, s.pgServer.HBADebugFn(), s.execCfg.SQLStatusServer)
mux.Handle("/", debugServer)
mux.Handle("/_status/", gwMux)
mux.HandleFunc("/health", func(w http.ResponseWriter, req *http.Request) {
Expand Down
2 changes: 2 additions & 0 deletions pkg/ui/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ assets.oss.installed
# Generated intermediates
.cache-loader
dist*/**
dist_vendor/**
!dist_vendor/.gitkeep
!dist*/_empty_assets/assets/index.html
!distccl/assets/.gitkeep
!distccl/distccl.go
Expand Down
19 changes: 18 additions & 1 deletion pkg/ui/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")

go_library(
name = "ui",
srcs = ["ui.go"],
srcs = [
"tracing_ui.go",
"ui.go",
],
embedsrcs = [
"dist_vendor/list.min.js",
"templates/tracing/html_template.html",
"dist_vendor/.gitkeep",
],
importpath = "github.com/cockroachdb/cockroach/pkg/ui",
visibility = ["//visibility:public"],
deps = [
Expand Down Expand Up @@ -35,3 +44,11 @@ EOF
""",
visibility = ["//pkg/ui:__subpackages__"],
)

genrule(
name = "listjs",
srcs = ["@npm//:node_modules/list.js/dist/list.min.js"],
outs = ["dist_vendor/list.min.js"],
cmd = "cp ./$(location @npm//:node_modules/list.js/dist/list.min.js) $@",
tools = ["@npm//list.js"],
)
Empty file added pkg/ui/dist_vendor/.gitkeep
Empty file.
140 changes: 140 additions & 0 deletions pkg/ui/templates/tracing/html_template.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<style>
.sort.asc, .sort.desc {
background-color: yellow;
}
.sort.asc::after {
content: "\002B06";
padding-left: 3px;
}
.sort.desc::after {
content: "\002B07";
padding-left: 3px
}

.link-button {
background: none!important;
border: none;
padding: 0!important;
font-family: arial, sans-serif;
color: #069;
text-decoration: underline;
cursor: pointer;
}

</style>
</head>

<script src="/debug/assets/list.min.js" type="text/javascript"></script>

<a href="?snap=new">Take a snapshot of current operations</a>
<div style="float:right">
Stored snapshots (ID: capture time):
{{$id := .SnapshotID}}
{{range $i, $s := .AllSnapshots}}
<span style="{{if ne $i 0}}margin-left:0.5em;{{end}}">
{{if eq $s.ID $id}}
[current] {{$s.ID}}: {{formatTimeNoMillis .CapturedAt}}
{{else}}
<a href="?snap={{$s.ID}}">{{$s.ID}}: {{formatTimeNoMillis .CapturedAt}}</a>
{{end}}
</span>
{{end}}
</div>

<p>Spans currently open: {{len .SpansList.Spans}}. Snapshot captured at: {{formatTime .CapturedAt}} UTC. Page generated at: {{formatTime .Now}} UTC.</p>
{{if ne .Err nil}}
<p><b>There was an error producing this snapshot; it might be incomplete: {{.Err}}</b></p>
{{end}}

<div id="spans-list">
<table>
<thead>
<input class="search" placeholder="Search" />
<tr>
<th>
<button class="sort link-button" data-sort="operation"> Operation </button>
</th>
<th>
<button class="sort link-button" data-sort="startTimeMicros"> Started at </button>
</th>
<th>
<button class="sort link-button" data-sort="goroutineID"> Goroutine ID </button>
</th>
</tr>
</thead>
<tbody class="list">
<!-- This will be populated by spansList. -->
</tbody>
</table>
</div>

<script>

// Some fields are strings because JS doesn't do 64-bit numbers properly.
var values = [
{{$capturedAt := .CapturedAt}}
{{$stacks := .SpansList.Stacks}}
{{range .SpansList.Spans}}
{
operation: {{.Operation}},
spanID: "{{.SpanID}}",
startTime: {{formatTime .StartTime}},
relativeTime: {{since .StartTime $capturedAt}},
startTimeMicros: {{timeRaw .StartTime}},
goroutineID: {{.GoroutineID}},
traceID: "{{.TraceID}}",
stack: {{index $stacks .GoroutineID}}
},
{{end}}
];

function toggleStackVisibility(spanID) {
var div = document.getElementById("stack-"+spanID);
if (div.style.display === "none") {
div.style.display = "block";
} else {
div.style.display = "none";
}
};


// openTrace opens the requested trace in a new window.
function openTrace(traceID) {
const urlParams = new URLSearchParams(window.location.search);
const snapID = urlParams.get('snap');
window.open(`show-trace?snap=${snapID}&trace=${traceID}`, "_blank");
}

var options = {
valueNames: ['operation', 'startTime', 'relativeTime', 'goroutineID', 'spanID', 'stack', 'traceID'],
item: function (values) {
return `
<tr>
<td>
${values.operation} <a href="javascript:openTrace('${values.traceID}')">[+]</a>
</td>
<td>
<div style="min-width:12em">
<span class="startTime"></span>
<span class="relativeTime" style="float:right"></span>
</div>
</td>>
<td style="text-align:center">
<div class="goroutineID link-button" onclick="toggleStackVisibility('${values.spanID}');"></div>
<div style="display:none; text-align:left;" id="stack-${values.spanID}"><pre>${values.stack}</pre></div>
</td>
</tr>`;
},
};

// Create the list of spans.
var spansList = new List('spans-list', options, values);
// Start with spans sorted newest first.
spansList.sort('startTimeMicros', {order: "desc"});

</script>
</html>
36 changes: 36 additions & 0 deletions pkg/ui/tracing_ui.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright 2021 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

package ui

import (
"embed"
"io/fs"
)

// This file deals with embedding assets used by /debug/tracez.

//go:embed dist_vendor/*
var vendorFiles embed.FS

// VendorFS exposes the list.js package.
var VendorFS fs.FS

//go:embed templates/tracing/html_template.html
// SpansTableTemplateSrc contains a template used by /debug/tracez
var SpansTableTemplateSrc string

func init() {
var err error
VendorFS, err = fs.Sub(vendorFiles, "dist_vendor")
if err != nil {
panic(err)
}
}
1 change: 1 addition & 0 deletions pkg/ui/workspaces/db-console/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ DEPENDENCIES = [
"@npm//highlight.js",
"@npm//lodash",
"@npm//long",
"@npm//list.js",
"@npm//mini-create-react-context",
"@npm//moment",
"@npm//nvd3",
Expand Down
1 change: 1 addition & 0 deletions pkg/ui/workspaces/db-console/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
"connected-react-router": "^6.9.1",
"create-react-context": "^0.3.0",
"highlight.js": "^10.6.0",
"list.js": "^2.3.1",
"lodash": "^4.17.21",
"long": "^4.0.0",
"mini-create-react-context": "^0.3.2",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ export default function Debug() {
</DebugTable>
<DebugTable heading="Tracing and Profiling Endpoints (local node only)">
<DebugTableRow title="Tracing">
<DebugTableLink name="Active operations" url="debug/tracez" />
<DebugTableLink name="Requests" url="debug/requests" />
<DebugTableLink name="Events" url="debug/events" />
<DebugTableLink
Expand Down
8 changes: 8 additions & 0 deletions pkg/ui/workspaces/db-console/webpack.app.js
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ module.exports = (env, argv) => {
new RemoveBrokenDependenciesPlugin(),
new CopyWebpackPlugin([
{ from: path.resolve(__dirname, "favicon.ico"), to: "favicon.ico" },
{
from: path.resolve(
!isBazelBuild ? __dirname : "",
!isBazelBuild ? "../.." : "",
"node_modules/list.js/dist/list.min.js",
),
to: path.resolve(__dirname, "../../dist_vendor/list.min.js"),
},
]),
// use WebpackBar instead of webpack dashboard to fit multiple webpack dev server outputs (db-console and cluster-ui)
new WebpackBar({
Expand Down
2 changes: 1 addition & 1 deletion pkg/ui/yarn-vendor
12 changes: 12 additions & 0 deletions pkg/ui/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -11461,6 +11461,13 @@ linkify-it@^2.0.0:
dependencies:
uc.micro "^1.0.1"

list.js@^2.3.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/list.js/-/list.js-2.3.1.tgz#48961989ffe52b0505e352f7a521f819f51df7e7"
integrity sha512-jnmm7DYpKtH3DxtO1E2VNCC9Gp7Wrp/FWA2JxQrZUhVJ2RCQBd57pCN6W5w6jpsfWZV0PCAbTX2NOPgyFeeZZg==
dependencies:
string-natural-compare "^2.0.2"

listr-silent-renderer@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e"
Expand Down Expand Up @@ -16399,6 +16406,11 @@ string-length@^3.1.0:
astral-regex "^1.0.0"
strip-ansi "^5.2.0"

string-natural-compare@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/string-natural-compare/-/string-natural-compare-2.0.3.tgz#9dbe1dd65490a5fe14f7a5c9bc686fc67cb9c6e4"
integrity sha512-4Kcl12rNjc+6EKhY8QyDVuQTAlMWwRiNbsxnVwBUKFr7dYPQuXVrtNU4sEkjF9LHY0AY6uVbB3ktbkIH4LC+BQ==

string-replace-webpack-plugin@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/string-replace-webpack-plugin/-/string-replace-webpack-plugin-0.1.3.tgz#73c657e759d66cfe80ae1e0cf091aa256d0e715c"
Expand Down
1 change: 1 addition & 0 deletions pkg/util/tracing/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_library(
"tags.go",
"test_utils.go",
"tracer.go",
"tracer_snapshots.go",
"utils.go",
],
importpath = "github.com/cockroachdb/cockroach/pkg/util/tracing",
Expand Down
Loading

0 comments on commit f87695e

Please sign in to comment.