diff --git a/doc/releasenotes/16_0_0_summary.md b/doc/releasenotes/16_0_0_summary.md index 691513461a7..386283ac9d8 100644 --- a/doc/releasenotes/16_0_0_summary.md +++ b/doc/releasenotes/16_0_0_summary.md @@ -33,6 +33,21 @@ In [PR #11097](https://github.com/vitessio/vitess/pull/11097) we introduced nati - A server restored to a point in time remains in `DRAINED` tablet type, and does not join the replication stream (thus, "frozen" in time). - It is possible to take incremental backups from different tablets. It is OK to have overlaps in incremental backup contents. The restore process chooses a valid path, and is valid as long as there are no gaps in the backed up binary log content. + +### Structured Logging Flag + +In [PR #11960](https://github.com/vitessio/vitess/pull/11960/) we introduced a new flag `structured-logging`. This flag stops the `glog` usage and use [`PlanetScale Log`](https://github.com/planetscale/log) instead. + +The flag is available in these components: + +- mysqlctl +- mysqlctld +- vtctld +- vtgate +- vtorc +- vttablet +- vttestserver + ### Breaking Changes #### Orchestrator Integration Deletion diff --git a/go.mod b/go.mod index b7a56cb336c..cb518ca0427 100644 --- a/go.mod +++ b/go.mod @@ -113,6 +113,9 @@ require ( github.com/bndr/gotabulate v1.1.2 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 github.com/openark/golib v0.0.0-20210531070646-355f37940af8 + github.com/planetscale/log v0.0.0-20221118170849-fb599bc35c50 + github.com/slok/noglog v0.2.0 + go.uber.org/zap v1.23.0 golang.org/x/exp v0.0.0-20221114191408-850992195362 ) @@ -185,7 +188,6 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.23.0 // indirect go4.org/intern v0.0.0-20220617035311-6925f38cc365 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect golang.org/x/exp/typeparams v0.0.0-20221114191408-850992195362 // indirect @@ -204,3 +206,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) + +replace github.com/google/glog => github.com/planetscale/noglog v0.2.1-0.20210421230640-bea75fcd2e8e diff --git a/go.sum b/go.sum index 73684d6541c..c1ade88b5f2 100644 --- a/go.sum +++ b/go.sum @@ -650,6 +650,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/planetscale/log v0.0.0-20221118170849-fb599bc35c50 h1:yf0iVpE57riOj2+cLr6Fy6qX6eCLc5RLK3Kc70apP6c= +github.com/planetscale/log v0.0.0-20221118170849-fb599bc35c50/go.mod h1:SE8Q9QtLD8tfq8bM7rGLJnnWfmxt6mTXGkfGbft1vJI= github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a h1:y0OpQ4+5tKxeh9+H+2cVgASl9yMZYV9CILinKOiKafA= github.com/planetscale/pargzip v0.0.0-20201116224723-90c7fc03ea8a/go.mod h1:GJFUzQuXIoB2Kjn1ZfDhJr/42D5nWOqRcIQVgCxTuIE= github.com/planetscale/vtprotobuf v0.3.0 h1:oMrOdDFHS1ADc0dHtC2EApxiM5xd0cQkZeibm0WgXiQ= @@ -713,6 +715,8 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sjmudd/stopwatch v0.1.1 h1:x45OvxFB5OtCkjvYtzRF5fWB857Jzjjk84Oyd5C5ebw= github.com/sjmudd/stopwatch v0.1.1/go.mod h1:BLw0oIQJ1YLXBO/q9ufK/SgnKBVIkC2qrm6uy78Zw6U= +github.com/slok/noglog v0.2.0 h1:1czu4l2EoJ8L92UwdSXXa1Y+c5TIjFAFm2P+mjej95E= +github.com/slok/noglog v0.2.0/go.mod h1:TfKxwpEZPT+UA83bQ6RME146k0MM4e8mwHLf6bhcGDI= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index 64581418662..f7e3c3fac8c 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -73,6 +73,7 @@ Global flags: --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice --socket_file string Local unix socket file to listen on --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_uid uint Tablet UID (default 41983) --v Level log level for V logs diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index e00ac7fac09..0061456b836 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -84,6 +84,7 @@ Usage of mysqlctld: --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice --socket_file string Local unix socket file to listen on --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_uid uint Tablet UID (default 41983) --v Level log level for V logs diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index f62908aa215..417e6de35b2 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -100,6 +100,7 @@ Usage of vtctld: --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_grpc_ca string the server ca to use to validate servers when connecting --tablet_grpc_cert string the cert to use to connect diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 1022b6ef614..43435c35085 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -156,6 +156,7 @@ Usage of vtgate: --statsd_sample_rate float Sample rate for statsd metrics (default 1) --stderrthreshold severity logs at or above this threshold go to stderr (default 1) --stream_buffer_size int the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --tablet_filters strings Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch. --tablet_grpc_ca string the server ca to use to validate servers when connecting --tablet_grpc_cert string the cert to use to connect diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index 68ffda88f1e..c373fd7062f 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -51,6 +51,7 @@ Usage of vtorc: --stats_drop_variables string Variables to be dropped from the list of exported variables. --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting --tablet_manager_grpc_cert string the cert to use to connect --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 1e12c2781e1..7b088dae906 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -279,6 +279,7 @@ Usage of vttablet: --statsd_sample_rate float Sample rate for statsd metrics (default 1) --stderrthreshold severity logs at or above this threshold go to stderr (default 1) --stream_health_buffer_size uint max streaming health entries to buffer per streaming health client (default 20) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --table-acl-config string path to table access checker config file; send SIGHUP to reload this file --table-acl-config-reload-interval duration Ticker to reload ACLs. Duration flag, format e.g.: 30s. Default: do not reload --table_gc_lifecycle string States for a DROP TABLE garbage collection cycle. Default is 'hold,purge,evac,drop', use any subset ('drop' implcitly always included) (default "hold,purge,evac,drop") diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 197ab0bdf2c..86ca3f8c434 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -98,6 +98,7 @@ Usage of vttestserver: --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --structured-logging whether to use structured logging (PlanetScale Log) logger or the original (glog) logger --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. --tablet_hostname string The hostname to use for the tablet otherwise it will be derived from OS' hostname (default "localhost") --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting diff --git a/go/vt/logutil/planetscale_logger.go b/go/vt/logutil/planetscale_logger.go new file mode 100644 index 00000000000..7133188df4f --- /dev/null +++ b/go/vt/logutil/planetscale_logger.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logutil + +import ( + pslog "github.com/planetscale/log" + noglog "github.com/slok/noglog" + + "vitess.io/vitess/go/vt/log" +) + +type PSLogger pslog.SugaredLogger + +// SetPlanetScaleLogger in-place noglog replacement with PlanetScale's logger. +func SetPlanetScaleLogger(conf *pslog.Config) (psLogger *pslog.SugaredLogger, err error) { + // Use the passed configuration instead of the default configuration + if conf != nil { + configLogger, err := conf.Build() + if err != nil { + return nil, err + } + psLogger = configLogger.Sugar() + } else { + psLogger = pslog.NewPlanetScaleSugarLogger() + } + + noglog.SetLogger(&noglog.LoggerFunc{ + DebugfFunc: func(f string, a ...interface{}) { psLogger.Debugf(f, a...) }, + InfofFunc: func(f string, a ...interface{}) { psLogger.Infof(f, a...) }, + WarnfFunc: func(f string, a ...interface{}) { psLogger.Warnf(f, a...) }, + ErrorfFunc: func(f string, a ...interface{}) { psLogger.Errorf(f, a...) }, + }) + + log.Flush = noglog.Flush + log.Info = noglog.Info + log.Infof = noglog.Infof + log.InfoDepth = noglog.InfoDepth + log.Warning = noglog.Warning + log.Warningf = noglog.Warningf + log.WarningDepth = noglog.WarningDepth + log.Error = noglog.Error + log.Errorf = noglog.Errorf + log.ErrorDepth = noglog.ErrorDepth + log.Exit = noglog.Exit + log.Exitf = noglog.Exitf + log.ExitDepth = noglog.ExitDepth + log.Fatal = noglog.Fatal + log.Fatalf = noglog.Fatalf + log.FatalDepth = noglog.FatalDepth + + return +} diff --git a/go/vt/logutil/planetscale_logger_test.go b/go/vt/logutil/planetscale_logger_test.go new file mode 100644 index 00000000000..9433a0ddca0 --- /dev/null +++ b/go/vt/logutil/planetscale_logger_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logutil + +import ( + "bytes" + "encoding/json" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + + pslog "github.com/planetscale/log" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + vtlog "vitess.io/vitess/go/vt/log" +) + +// MemorySink implements zap.Sink by writing all messages to a buffer. +// It's used to capture the logs. +type MemorySink struct { + *bytes.Buffer +} + +// Implement Close and Sync as no-ops to satisfy the interface. The Write +// method is provided by the embedded buffer. +func (s *MemorySink) Close() error { return nil } +func (s *MemorySink) Sync() error { return nil } + +func SetupLoggerWithMemSink() (sink *MemorySink, err error) { + // Create a sink instance, and register it with zap for the "memory" + // protocol. + sink = &MemorySink{new(bytes.Buffer)} + err = zap.RegisterSink("memory", func(*url.URL) (zap.Sink, error) { + return sink, nil + }) + if err != nil { + return nil, err + } + + testLoggerConf := pslog.NewPlanetScaleConfig(pslog.DetectEncoding(), pslog.InfoLevel) + testLoggerConf.OutputPaths = []string{"memory://"} + testLoggerConf.ErrorOutputPaths = []string{"memory://"} + _, err = SetPlanetScaleLogger(&testLoggerConf) + if err != nil { + return nil, err + } + + return +} + +func TestPSLogger_Replacing_glog(t *testing.T) { + type logMsg struct { + Level string `json:"level"` + Msg string `json:"msg"` + } + + type testCase struct { + name string + logLevel zapcore.Level + } + + dummyLogMessage := "testing log" + testCases := []testCase{ + {"log info", pslog.InfoLevel}, + {"log warn", pslog.WarnLevel}, + {"log error", pslog.ErrorLevel}, + } + + sink, err := SetupLoggerWithMemSink() + assert.NoError(t, err) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var loggingFunc func(format string, args ...interface{}) + var expectedLevel string + + switch tc.logLevel { + case zapcore.InfoLevel: + loggingFunc = vtlog.Infof + expectedLevel = "info" + case zapcore.ErrorLevel: + loggingFunc = vtlog.Errorf + expectedLevel = "error" + case zapcore.WarnLevel: + loggingFunc = vtlog.Warningf + expectedLevel = "warn" + } + + loggingFunc(dummyLogMessage) + + // Unmarshal the captured log. This means we're getting a struct log. + actualLog := logMsg{} + err = json.Unmarshal(sink.Bytes(), &actualLog) + assert.NoError(t, err) + // Reset the sink so that it'll contain one log per test case. + sink.Reset() + + assert.Equal(t, expectedLevel, actualLog.Level) + assert.Equal(t, dummyLogMessage, actualLog.Msg) + + }) + } +} diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index 4e5d582a2ae..1a2b7b1a81d 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -82,6 +82,7 @@ var ( onCloseTimeout = 10 * time.Second catchSigpipe bool maxStackSize = 64 * 1024 * 1024 + usePSLogger bool ) // RegisterFlags installs the flags used by Init, Run, and RunDefault. @@ -98,6 +99,8 @@ func RegisterFlags() { // pid_file.go fs.StringVar(&pidFile, "pid_file", pidFile, "If set, the process will write its pid to the named file, and delete it on graceful shutdown.") + // Logging + fs.BoolVar(&usePSLogger, "structured-logging", usePSLogger, "whether to use structured logging (PlanetScale Log) logger or the original (glog) logger") }) } @@ -322,6 +325,14 @@ func ParseFlags(cmd string) { os.Exit(0) } + if usePSLogger { + // Replace glog logger with PlanetScale logger + _, err := logutil.SetPlanetScaleLogger(nil) + if err != nil { + log.Exitf("error while setting the PlanetScale logger: %s", err) + } + } + args := fs.Args() if len(args) > 0 { _flag.Usage()