Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement the ability to more logically share level hierarchies #134

Merged
merged 3 commits into from
Dec 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 77 additions & 12 deletions intlogger.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,16 @@ type intLogger struct {
writer *writer
level *int32

// The value of curEpoch the last time we performed the level sync process
ownEpoch uint64

// Shared amongst all the loggers created in this hierachy, used to determine
// if the level sync process should be run by comparing it with ownEpoch
curEpoch *uint64

// The logger this one was created from. Only set when syncParentLevel is set
parent *intLogger

headerColor ColorOption
fieldColor ColorOption

Expand All @@ -88,6 +98,7 @@ type intLogger struct {

// create subloggers with their own level setting
independentLevels bool
syncParentLevel bool

subloggerHook func(sub Logger) Logger
}
Expand Down Expand Up @@ -129,9 +140,9 @@ func newLogger(opts *LoggerOptions) *intLogger {
}

var (
primaryColor ColorOption = ColorOff
headerColor ColorOption = ColorOff
fieldColor ColorOption = ColorOff
primaryColor = ColorOff
headerColor = ColorOff
fieldColor = ColorOff
)
switch {
case opts.ColorHeaderOnly:
Expand All @@ -152,8 +163,10 @@ func newLogger(opts *LoggerOptions) *intLogger {
mutex: mutex,
writer: newWriter(output, primaryColor),
level: new(int32),
curEpoch: new(uint64),
exclude: opts.Exclude,
independentLevels: opts.IndependentLevels,
syncParentLevel: opts.SyncParentLevel,
headerColor: headerColor,
fieldColor: fieldColor,
subloggerHook: opts.SubloggerHook,
Expand Down Expand Up @@ -194,7 +207,7 @@ const offsetIntLogger = 3
// Log a message and a set of key/value pairs if the given level is at
// or more severe that the threshold configured in the Logger.
func (l *intLogger) log(name string, level Level, msg string, args ...interface{}) {
if level < Level(atomic.LoadInt32(l.level)) {
if level < l.GetLevel() {
return
}

Expand Down Expand Up @@ -597,7 +610,7 @@ func (l *intLogger) logJSON(t time.Time, name string, level Level, msg string, a
vals := l.jsonMapEntry(t, name, level, msg)
args = append(l.implied, args...)

if args != nil && len(args) > 0 {
if len(args) > 0 {
if len(args)%2 != 0 {
cs, ok := args[len(args)-1].(CapturedStacktrace)
if ok {
Expand Down Expand Up @@ -718,27 +731,27 @@ func (l *intLogger) Error(msg string, args ...interface{}) {

// Indicate that the logger would emit TRACE level logs
func (l *intLogger) IsTrace() bool {
return Level(atomic.LoadInt32(l.level)) == Trace
return l.GetLevel() == Trace
}

// Indicate that the logger would emit DEBUG level logs
func (l *intLogger) IsDebug() bool {
return Level(atomic.LoadInt32(l.level)) <= Debug
return l.GetLevel() <= Debug
}

// Indicate that the logger would emit INFO level logs
func (l *intLogger) IsInfo() bool {
return Level(atomic.LoadInt32(l.level)) <= Info
return l.GetLevel() <= Info
}

// Indicate that the logger would emit WARN level logs
func (l *intLogger) IsWarn() bool {
return Level(atomic.LoadInt32(l.level)) <= Warn
return l.GetLevel() <= Warn
}

// Indicate that the logger would emit ERROR level logs
func (l *intLogger) IsError() bool {
return Level(atomic.LoadInt32(l.level)) <= Error
return l.GetLevel() <= Error
}

const MissingKey = "EXTRA_VALUE_AT_END"
Expand Down Expand Up @@ -854,12 +867,62 @@ func (l *intLogger) resetOutput(opts *LoggerOptions) error {
// Update the logging level on-the-fly. This will affect all subloggers as
// well.
func (l *intLogger) SetLevel(level Level) {
atomic.StoreInt32(l.level, int32(level))
if !l.syncParentLevel {
atomic.StoreInt32(l.level, int32(level))
return
}

nsl := new(int32)
*nsl = int32(level)

l.level = nsl

l.ownEpoch = atomic.AddUint64(l.curEpoch, 1)
}

func (l *intLogger) searchLevelPtr() *int32 {
p := l.parent

ptr := l.level

max := l.ownEpoch

for p != nil {
if p.ownEpoch > max {
max = p.ownEpoch
ptr = p.level
}

p = p.parent
}

return ptr
}

// Returns the current level
func (l *intLogger) GetLevel() Level {
return Level(atomic.LoadInt32(l.level))
// We perform the loads immediately to keep the CPU pipeline busy, which
// effectively makes the second load cost nothing. Once loaded into registers
// the comparison returns the already loaded value. The comparison is almost
// always true, so the branch predictor should hit consistently with it.
var (
curEpoch = atomic.LoadUint64(l.curEpoch)
level = Level(atomic.LoadInt32(l.level))
own = l.ownEpoch
)

if curEpoch == own {
return level
}

// Perform the level sync process. We'll avoid doing this next time by seeing the
// epoch as current.

ptr := l.searchLevelPtr()
l.level = ptr
l.ownEpoch = curEpoch

return Level(atomic.LoadInt32(ptr))
}

// Create a *log.Logger that will send it's data through this Logger. This
Expand Down Expand Up @@ -912,6 +975,8 @@ func (l *intLogger) copy() *intLogger {
if l.independentLevels {
sl.level = new(int32)
*sl.level = *l.level
} else if l.syncParentLevel {
sl.parent = l
}

return &sl
Expand Down
18 changes: 18 additions & 0 deletions logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,24 @@ type LoggerOptions struct {
// will not affect the parent or sibling loggers.
IndependentLevels bool

// When set, changing the level of a logger effects only it's direct sub-loggers
// rather than all sub-loggers. For example:
// a := logger.Named("a")
// a.SetLevel(Error)
// b := a.Named("b")
// c := a.Named("c")
// b.GetLevel() => Error
// c.GetLevel() => Error
// b.SetLevel(Info)
// a.GetLevel() => Error
// b.GetLevel() => Info
// c.GetLevel() => Error
// a.SetLevel(Warn)
// a.GetLevel() => Warn
// b.GetLevel() => Warn
// c.GetLevel() => Warn
Comment on lines +318 to +321
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 I find this behaviour surprising.

My expectation would be that if some sub-logger (in this case b) has had its level specifically overridden (e.g. by an operator enabling more detailed level on a sub-component), than a subsequent change to the global level should not override the more specific, explicitly set level. It should cascade to all sub-loggers that haven't had an explicitly set level though.

I'm thinking of a UX where a peice of software allows operators to specify a targetted log level e.g via ENV var. Something like FOO_SUBCOMPONENT_LOG=TRACE while investigating an issue.

If they subsequently choose to reconfigure the application e.g. via config file to global log at DEBUG instead of INFO I'd expect that to apply to everything except the component that has already been overridden.

In this example, unlike the comment here, the override is for more detail. There could be an argument that you might want different behaviour depending on whether the override is for more or less detail, but I feel like it's less surprising in all cases to continue to honour more specific sub-logger overrides when changing the parent/global level.

Is there a use-case I'm not seeing where this behavior seems better?

Copy link
Contributor

@mpalmi mpalmi Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Evan and I chatted about this briefly and we contemplated the principle of least surprise. I don't feel strongly either way, but I think there are a couple of things to consider for Vault's implementation -- specifically around the distinction between /sys/loggers and your environment variable example.

In my mind, our implementation is slightly different from the environment variable case, since logger level state (at least for Vault) is entirely in program memory. With environment variables, you have state outside the scope of the program that would be honored on restart; whereas in Vault, that's not the case. I thought it would be less surprising to keep reload and restart behavior consistent and advertise the fact that /sys/loggers is really for transient maintenance tasks.

The flip side of that is the maintenance tasks might include reload, at which point this UX could become burdensome (edit: This may not actually be true. I think that log level is only sync'ed on reload if an operator changes the server's HCL, so someone would have to intentionally reset all levels.)

Like I said, I don't feel too strongly either way, but just some thoughts to consider.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Funny you ask this, I asked @mpalmi the same question. Effectively the idea here is that parent logger changes win over any previous decisions the sub-loggers made. @mpalmi thoughts on this? I think this is probably the biggest behavior that we need to sort out.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The idea put forth was that if, at runtime, a decision to put a hierarchy into TRACE mode is requested, everything in the hierarchy should get that change. If a specific component in hierarchy had been set to ERROR at program start, the desire to set the whole hierachry into TRACE would need to override ERROR.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@banks In your specific example, the user would have to be careful to configure the toplevel first, then components down the hierarchy second. In that way, they could achieve the same results.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't feel super strongly. At least to me I feel like I'd expect more specific override to take precedence and yes - for it's whole subtree. I've not thought through the actual use-cases or UX of vault /sys/loggers though.

But I think if I'd bothered to make an API call to set a logging level on a specific component, I'd not expect a general change of the logging level via some other mechanism to override that. I don't think I'd lose sleep over it, but it at least wouldn't be my expectation 🤷 .

If you thought this through Mike and this is the preferred UX then I'm fine as long as we document that clearly (both here and in our product docs etc.)

Copy link
Member

@banks banks Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

forever detached from changes by a?

Actually that's a bit more nuanced. I guess what I'd expect would be that if the component specific override for b is removed, then b and it's whole subtree are "re-attached" to the parent and start to respect that setting again.

I have no idea if anyone cares enough to warrant implementing that though! It would probably need an API change too so that it was possible to set a sub-logger back to "follow parent" level rather than some specific level...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, the ability to specify "hey, this is an override" and then "hey, reset this back to not an override" isn't available (and it would be probably a bit confusing).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I put in a vote for "parent always changing level for children is less surprising".

It's possible to achieve the detached logging levels with IndependentLevels and some use of state + sublogger hooks (I have a related poc in Consul: hashicorp/consul#16665)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, the current approach seems to be the least surprising -- especially if overrides cannot be reset.

SyncParentLevel bool

// SubloggerHook registers a function that is called when a sublogger via
// Named, With, or ResetNamed is created. If defined, the function is passed
// the newly created Logger and the returned Logger is returned from the
Expand Down
82 changes: 82 additions & 0 deletions logger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,88 @@ func TestLogger(t *testing.T) {
t.Fatal("output from disabled logger:", str)
}
})

t.Run("sub-loggers levels don't bubble upward", func(t *testing.T) {
var buf bytes.Buffer

logger := New(&LoggerOptions{
Name: "root",
Output: &buf,
SyncParentLevel: true,
})

another := logger.Named("sublogger")
another.SetLevel(Error)

logger.Info("this is test")
str := buf.String()
dataIdx := strings.IndexByte(str, ' ')
rest := str[dataIdx+1:]
assert.Equal(t, "[INFO] root: this is test\n", rest)

buf.Reset()

a := logger.Named("a")
b := a.Named("b")
c := a.Named("c")

a.SetLevel(Error)

b.Info("this is a test")

require.Empty(t, buf.String())

b.SetLevel(Info)

assert.Equal(t, Error, a.GetLevel())

a.SetLevel(Error)

assert.Equal(t, Error, b.GetLevel())

assert.Equal(t, Error, c.GetLevel())

// Make sure that setting a sibling logger doesn't confuse
// when b had previously had it's own level.
c.SetLevel(Info)

assert.Equal(t, Error, b.GetLevel())
})

t.Run("level sync example", func(t *testing.T) {
var buf bytes.Buffer

logger := New(&LoggerOptions{
Name: "root",
Output: &buf,
SyncParentLevel: true,
})

s := assert.New(t)

a := logger.Named("a")
a.SetLevel(Error)
b := a.Named("b")
c := a.Named("c")
s.Equal(Error, b.GetLevel())
s.Equal(Error, c.GetLevel())

b.SetLevel(Info)
s.Equal(Error, a.GetLevel())
s.Equal(Info, b.GetLevel())
s.Equal(Error, c.GetLevel())

a.SetLevel(Warn)
s.Equal(Warn, a.GetLevel())
s.Equal(Warn, b.GetLevel())
s.Equal(Warn, c.GetLevel())

logger.SetLevel(Trace)
s.Equal(Trace, logger.GetLevel())
s.Equal(Trace, a.GetLevel())
s.Equal(Trace, b.GetLevel())
s.Equal(Trace, c.GetLevel())
})
}

func TestLogger_leveledWriter(t *testing.T) {
Expand Down