From 8ba38063a360999832079f63af852f549f2bba25 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Thu, 26 Nov 2020 11:21:08 +0100 Subject: [PATCH] logconfig,cli: new command `debug check-log-config` Release note (cli change): A new command `cockroch debug check-log-config` prints out the logging configuration that results from the provided combination of `--store`, `--log` and other logging-related flags on the command line. --- pkg/cli/BUILD.bazel | 2 + pkg/cli/debug.go | 4 + pkg/cli/debug_logconfig.go | 54 ++++++ pkg/cli/flags.go | 4 + pkg/cli/start.go | 2 +- pkg/util/log/logconfig/BUILD.bazel | 2 + pkg/util/log/logconfig/export.go | 257 +++++++++++++++++++++++++ pkg/util/log/logconfig/export_test.go | 50 +++++ pkg/util/log/logconfig/testdata/export | 77 ++++++++ 9 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 pkg/cli/debug_logconfig.go create mode 100644 pkg/util/log/logconfig/export.go create mode 100644 pkg/util/log/logconfig/export_test.go create mode 100644 pkg/util/log/logconfig/testdata/export diff --git a/pkg/cli/BUILD.bazel b/pkg/cli/BUILD.bazel index f9948ed977bd..d51618418304 100644 --- a/pkg/cli/BUILD.bazel +++ b/pkg/cli/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "cpuprofile.go", "debug.go", "debug_check_store.go", + "debug_logconfig.go", "debug_merge_logs.go", "debug_synctest.go", "decode.go", @@ -201,6 +202,7 @@ go_library( "//vendor/google.golang.org/grpc", "//vendor/google.golang.org/grpc/codes", "//vendor/google.golang.org/grpc/status", + "//vendor/gopkg.in/yaml.v2:yaml_v2", ] + select({ "@io_bazel_rules_go//go/platform:aix": [ "//vendor/golang.org/x/sys/unix", diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go index 9a2b0de963c9..3087b4665a84 100644 --- a/pkg/cli/debug.go +++ b/pkg/cli/debug.go @@ -1167,6 +1167,7 @@ var DebugCmdsForRocksDB = []*cobra.Command{ // All other debug commands go here. var debugCmds = append(DebugCmdsForRocksDB, debugBallastCmd, + debugCheckLogConfigCmd, debugDecodeKeyCmd, debugDecodeValueCmd, debugDecodeProtoCmd, @@ -1267,4 +1268,7 @@ func init() { f = debugDecodeProtoCmd.Flags() f.StringVar(&debugDecodeProtoName, "schema", "cockroach.sql.sqlbase.Descriptor", "fully qualified name of the proto to decode") + + f = debugCheckLogConfigCmd.Flags() + f.Var(&debugLogChanSel, "only-channels", "selection of channels to include in the output diagram.") } diff --git a/pkg/cli/debug_logconfig.go b/pkg/cli/debug_logconfig.go new file mode 100644 index 000000000000..34531bf8577f --- /dev/null +++ b/pkg/cli/debug_logconfig.go @@ -0,0 +1,54 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "context" + "fmt" + + "github.com/cockroachdb/cockroach/pkg/util/log/logconfig" + "github.com/cockroachdb/errors" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" +) + +var debugCheckLogConfigCmd = &cobra.Command{ + Use: "check-log-config", + Short: "test the log config passed via --log", + RunE: runDebugCheckLogConfig, +} + +var debugLogChanSel logconfig.ChannelList + +func runDebugCheckLogConfig(cmd *cobra.Command, args []string) error { + if err := setupLogging(context.Background(), cmd, + true /* isServerCmd */, false /* applyconfig */); err != nil { + return err + } + if cliCtx.ambiguousLogDir { + fmt.Fprintf(stderr, "warning: ambiguous configuration, consider overriding the logging directory\n") + } + + c := cliCtx.logConfig + r, err := yaml.Marshal(&c) + if err != nil { + return errors.Wrap(err, "printing configuration") + } + + fmt.Println("# configuration after validation:") + fmt.Println(string(r)) + + _, key := c.Export(debugLogChanSel) + fmt.Println("# graphical diagram URL:") + fmt.Printf("http://www.plantuml.com/plantuml/uml/%s\n", key) + + return nil +} diff --git a/pkg/cli/flags.go b/pkg/cli/flags.go index ec0688a3fc89..8f6fec9dfc38 100644 --- a/pkg/cli/flags.go +++ b/pkg/cli/flags.go @@ -792,6 +792,10 @@ func init() { boolFlag(f, &debugCtx.sizes, cliflags.Sizes) stringFlag(f, &debugCtx.decodeAsTableDesc, cliflags.DecodeAsTable) } + { + f := debugCheckLogConfigCmd.Flags() + varFlag(f, &serverCfg.Stores, cliflags.Store) + } { f := debugRangeDataCmd.Flags() boolFlag(f, &debugCtx.replicated, cliflags.Replicated) diff --git a/pkg/cli/start.go b/pkg/cli/start.go index 0f98e7c12a3d..41bf6bdd20c8 100644 --- a/pkg/cli/start.go +++ b/pkg/cli/start.go @@ -104,7 +104,7 @@ var serverCmds = append(StartCmds, mtStartSQLCmd) // customLoggingSetupCmds lists the commands that call setupLogging() // after other types of configuration. -var customLoggingSetupCmds = append(serverCmds, demoCmd) +var customLoggingSetupCmds = append(serverCmds, debugCheckLogConfigCmd, demoCmd) func initBlockProfile() { // Enable the block profile for a sample of mutex and channel operations. diff --git a/pkg/util/log/logconfig/BUILD.bazel b/pkg/util/log/logconfig/BUILD.bazel index 338d19f79714..2915efaf9e3c 100644 --- a/pkg/util/log/logconfig/BUILD.bazel +++ b/pkg/util/log/logconfig/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "config.go", "doc.go", + "export.go", "validate.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/util/log/logconfig", @@ -21,6 +22,7 @@ go_test( name = "logconfig_test", srcs = [ "config_test.go", + "export_test.go", "validate_test.go", ], data = glob(["testdata/**"]), diff --git a/pkg/util/log/logconfig/export.go b/pkg/util/log/logconfig/export.go new file mode 100644 index 000000000000..ab82d1dca0f3 --- /dev/null +++ b/pkg/util/log/logconfig/export.go @@ -0,0 +1,257 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package logconfig + +import ( + "bytes" + "compress/flate" + "fmt" + "io" + "sort" + + "github.com/cockroachdb/cockroach/pkg/util/log/logpb" +) + +// Export exports the configuration in the PlantUML synax. +// onlyChans, if non-empty, restricts the output to only +// those rules that target the specified channels. +func (c *Config) Export(onlyChans ChannelList) (string, string) { + chanSel := ChannelList{Channels: channelValues} + if len(onlyChans.Channels) > 0 { + chanSel = onlyChans + } + + var buf bytes.Buffer + buf.WriteString("@startuml\nleft to right direction\n") + + // Export the channels. + buf.WriteString("component sources {\n") + for _, ch := range chanSel.Channels { + fmt.Fprintf(&buf, "() %s\n", ch) + } + buf.WriteString("cloud stray as \"stray\\nerrors\"\n}\n") + + // The process stderr stream. + buf.WriteString("queue stderr\n") + + // links collects the relationships. We need to collect them and + // print them at the end because plantUML does not support + // interleaving box and arrow declarations. + links := []string{} + + // When a channel receives additional processing inside the logging + // package, e.g. filtering or removing redaction markers, we add + // additional boxes in a "process" component to indicate this. + pNum := 1 + process := func(target string, cc CommonSinkConfig) (res string, processing, links []string) { + const defaultArrow = "-->" + // The arrow is customized for the last step before the final + // destination of messages, to indicate the criticality + // ("exit-on-error"). A dotted arrow indicates that errors are + // tolerated. + arrow := defaultArrow + if !*cc.Criticality { + arrow = "..>" + } + // Introduce a "strip" box if the redactable flag is set. + if !*cc.Redactable { + pkey := fmt.Sprintf("p__%d", pNum) + pNum++ + processing = append(processing, fmt.Sprintf("card %s as \"strip\"", pkey)) + links = append(links, fmt.Sprintf("%s %s %s", pkey, arrow, target)) + target = pkey + arrow = defaultArrow + } + // Introduce a "redact" box if the redat flag is set. + if *cc.Redact { + pkey := fmt.Sprintf("p__%d", pNum) + pNum++ + processing = append(processing, fmt.Sprintf("card %s as \"redact\"", pkey)) + links = append(links, fmt.Sprintf("%s --> %s", pkey, target)) + target = pkey + arrow = defaultArrow + } + // Introduce the format box. + { + pkey := fmt.Sprintf("p__%d", pNum) + pNum++ + processing = append(processing, fmt.Sprintf("card %s as \"format:%s\"", pkey, *cc.Format)) + links = append(links, fmt.Sprintf("%s --> %s", pkey, target)) + target = pkey + arrow = defaultArrow + } + // Introduce a "filter" box if the filter severity is different from INFO. + if cc.Filter != logpb.Severity_INFO { + pkey := fmt.Sprintf("p__%d", pNum) + pNum++ + processing = append(processing, fmt.Sprintf("card %s as \"filter:%s\"", pkey, cc.Filter.String()[:1])) + links = append(links, fmt.Sprintf("%s --> %s", pkey, target)) + target = pkey + arrow = defaultArrow + } + return target, processing, links + } + + // processing retains the processing rules. + processing := []string{} + + // Export the files. We are grouping per directory. + // + // folderNames collects the directory names. + folderNames := []string{} + // folders map each directory to a list of files within. + folders := map[string][]string{} + fileNum := 1 + for _, fn := range c.Sinks.sortedFileGroupNames { + fc := c.Sinks.FileGroups[fn] + if fc.Filter == logpb.Severity_NONE { + // This file is not collecting anything. Skip it. + continue + } + fileKey := fmt.Sprintf("f%d", fileNum) + fileNum++ + target := fileKey + + var syncproc, synclink []string + if *fc.SyncWrites { + skey := fmt.Sprintf("sync%d", fileNum) + fileNum++ + syncproc = append(syncproc, fmt.Sprintf("card %s as \"sync\"", skey)) + synclink = append(synclink, fmt.Sprintf("%s --> %s", skey, target)) + fileNum++ + target = skey + } + + target, thisprocs, thislinks := process(target, fc.CommonSinkConfig) + hasLink := false + for _, ch := range fc.Channels.Channels { + if !chanSel.HasChannel(ch) { + continue + } + hasLink = true + links = append(links, fmt.Sprintf("%s --> %s", ch, target)) + } + + if hasLink { + processing = append(processing, syncproc...) + processing = append(processing, thisprocs...) + links = append(links, thislinks...) + links = append(links, synclink...) + fileName := "cockroach.log" + if fc.prefix != "default" { + fileName = fmt.Sprintf("cockroach-%s.log", fc.prefix) + } + if _, ok := folders[*fc.Dir]; !ok { + folderNames = append(folderNames, *fc.Dir) + } + folders[*fc.Dir] = append(folders[*fc.Dir], + fmt.Sprintf("file %s as \"%s\"", fileKey, fileName)) + } + } + + // If the fd2 capture is enabled, represent it. + if c.CaptureFd2.Enable { + dir := *c.CaptureFd2.Dir + if _, ok := folders[dir]; !ok { + folderNames = append(folderNames, dir) + } + fileName := "cockroach-stderr.log" + folders[dir] = append(folders[dir], + fmt.Sprintf("file stderrfile as \"%s\"", fileName)) + links = append(links, "stray --> stderrfile") + } else { + links = append(links, "stray --> stderr") + } + + // Export the stderr redirects. + if c.Sinks.Stderr.Filter != logpb.Severity_NONE { + target, thisprocs, thislinks := process("stderr", c.Sinks.Stderr.CommonSinkConfig) + + hasLink := false + for _, ch := range c.Sinks.Stderr.Channels.Channels { + if !chanSel.HasChannel(ch) { + continue + } + hasLink = true + links = append(links, fmt.Sprintf("%s --> %s", ch, target)) + } + if hasLink { + links = append(links, thislinks...) + processing = append(processing, thisprocs...) + } + } + + // Represent the processing stages, if any. + if len(processing) > 0 { + for _, p := range processing { + fmt.Fprintf(&buf, "%s\n", p) + } + } + + // Represent the files, if any. + sort.Strings(folderNames) + if len(folderNames) > 0 { + buf.WriteString("artifact files {\n") + for _, fd := range folderNames { + fmt.Fprintf(&buf, " folder \"%s\" {\n", fd) + for _, fdecl := range folders[fd] { + fmt.Fprintf(&buf, " %s\n", fdecl) + } + buf.WriteString(" }\n") + } + buf.WriteString("}\n") + } + + // Export the relationships. + for _, l := range links { + fmt.Fprintf(&buf, "%s\n", l) + } + + // We are done! + buf.WriteString("@enduml\n") + uml := buf.String() + return uml, exportKey(uml) +} + +// exportKey returns the PlantUML encoded key for this diagram. +// +// Documentation: https://plantuml.com/text-encoding +func exportKey(uml string) string { + // DEFLATE + var b bytes.Buffer + w, _ := flate.NewWriter(&b, flate.BestCompression) + _, _ = io.WriteString(w, uml) + _ = w.Close() + compressed := b.Bytes() + + // BASE64 + inputLength := len(compressed) + for i := 0; i < 3-inputLength%3; i++ { + compressed = append(compressed, byte(0)) + } + + b = bytes.Buffer{} + for i := 0; i < inputLength; i += 3 { + b1, b2, b3, b4 := compressed[i], compressed[i+1], compressed[i+2], byte(0) + + b4 = b3 & 0x3f + b3 = ((b2 & 0xf) << 2) | (b3 >> 6) + b2 = ((b1 & 0x3) << 4) | (b2 >> 4) + b1 = b1 >> 2 + + for _, n := range []byte{b1, b2, b3, b4} { + // PlantUML uses a special base64 dictionary. + const base64 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_" + b.WriteByte(base64[n]) + } + } + return b.String() +} diff --git a/pkg/util/log/logconfig/export_test.go b/pkg/util/log/logconfig/export_test.go new file mode 100644 index 000000000000..72e37ac059dd --- /dev/null +++ b/pkg/util/log/logconfig/export_test.go @@ -0,0 +1,50 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package logconfig + +import ( + "bytes" + "fmt" + "testing" + + "github.com/cockroachdb/datadriven" + "gopkg.in/yaml.v2" +) + +func TestExport(t *testing.T) { + datadriven.RunTest(t, "testdata/export", func(t *testing.T, d *datadriven.TestData) string { + var onlyChans ChannelList + if d.HasArg("only-channels") { + var s string + d.ScanArgs(t, "only-channels", &s) + chs, err := parseChannelList(s) + if err != nil { + t.Fatal(err) + } + onlyChans.Channels = chs + } + + c := DefaultConfig() + if err := yaml.UnmarshalStrict([]byte(d.Input), &c); err != nil { + t.Fatal(err) + } + defaultDir := "/default-dir" + var buf bytes.Buffer + if err := c.Validate(&defaultDir); err != nil { + t.Fatal(err) + } else { + uml, key := c.Export(onlyChans) + buf.WriteString(uml) + fmt.Fprintf(&buf, "# http://www.plantuml.com/plantuml/uml/%s\n", key) + } + return buf.String() + }) +} diff --git a/pkg/util/log/logconfig/testdata/export b/pkg/util/log/logconfig/testdata/export new file mode 100644 index 000000000000..2e5d54857879 --- /dev/null +++ b/pkg/util/log/logconfig/testdata/export @@ -0,0 +1,77 @@ +# Default config +yaml +---- +@startuml +left to right direction +component sources { +() DEV +() STORAGE +() SESSIONS +() SENSITIVE_ACCESS +() SQL_EXEC +() SQL_PERF +() SQL_INTERNAL_PERF +cloud stray as "stray\nerrors" +} +queue stderr +card p__1 as "format:crdb-v1" +artifact files { + folder "/default-dir" { + file f1 as "cockroach.log" + file stderrfile as "cockroach-stderr.log" + } +} +DEV --> p__1 +STORAGE --> p__1 +SESSIONS --> p__1 +SENSITIVE_ACCESS --> p__1 +SQL_EXEC --> p__1 +SQL_PERF --> p__1 +SQL_INTERNAL_PERF --> p__1 +p__1 --> f1 +stray --> stderrfile +@enduml +# http://www.plantuml.com/plantuml/uml/L99FIyD04CNl-oc6dFGGfLSFqjAk4f1Kcr8y267TFsrmcwcp4q6atrsoczBab3VlFRApFtOUM15tHo-yTGoCGCt-m60QifeRR8N6umbRsp84x4ZR07_YvXPMSjT_gkrwitYMKSggAjPbbNHP5TjY9-l5SYchvBszrF9TBY_wLMwUBheejt9JBWPJU-mC12Rr2of05jL7QucGGYREuhkpdON0nX89hSZ0gQvdiUkGZeeVD9dF_6UM2KNSEAKPNEFZxk3G6qkGtHlhLESvDmrbVH0huD9_DEelGgKFTnxtsILCHqOvAUKf6BfdSHOhkOCyVuoBYO7Io1XOZPqfiL4oS9iwFQYfCs5uZIAPVd8paP3sm_KYOcvRqp-0_m000F__ + +# Capture everything to one file with sync and warnings only to stderr. +yaml only-channels=DEV,SESSIONS +sinks: + file-groups: + everything: + sync-writes: true + redact: true + channels: ALL + stderr: + filter: WARNING +---- +@startuml +left to right direction +component sources { +() DEV +() SESSIONS +cloud stray as "stray\nerrors" +} +queue stderr +card sync2 as "sync" +card p__1 as "redact" +card p__2 as "format:crdb-v1" +card p__3 as "format:crdb-v1-tty" +card p__4 as "filter:W" +artifact files { + folder "/default-dir" { + file f1 as "cockroach-everything.log" + file stderrfile as "cockroach-stderr.log" + } +} +DEV --> p__2 +SESSIONS --> p__2 +p__1 --> sync2 +p__2 --> p__1 +sync2 --> f1 +stray --> stderrfile +DEV --> p__4 +SESSIONS --> p__4 +p__3 --> stderr +p__4 --> p__3 +@enduml +# http://www.plantuml.com/plantuml/uml/R98nJ_Cm48Rt-nKdJzyt11JQgGFgq0uiC4Gg2r9bx7DhuThbVAaKeVvt5Bib89XYl-yJ9_V8oooQfJy42EG49I7xtLxGUYOZFaKmwN1CaQ9WJZqRolW1__xZQhqP7zswwnwU7Zim8VKMix0UK6TKPVKIYJbnLd26zvvwmYoMcC5ejfY7QEugF4IZQdZSRjkICLbjP4ehwH8Vj2mCszVcr4xjx8-s4HacObu97uHuyQn0itYdZQ3peGo5BWLBZEhMajDzaCPwLcDH47JrlqmoRvoqsJTq8Xvax-Fk9gITkd9rnBByoTVYmfxX3Alr1flclam7LvDJKbICko8AYeDBsKALDsvT2rLxGRy-_ltq-Q_Jvr2aJQz0KNHfPx2aQCTRyHa00F__