fix(spanner): fix negative values for max_in_use_sessions metrics #10449

 (#10508)

* fix(spanner): add debug log to print full stack trace when negative value happens

* skip decrementing num_in_use metric count when session is destroyed from healthchecks.

spanner/session.go

@@ -1191,6 +1191,11 @@ func (p *sessionPool) incNumInUseLocked(ctx context.Context) {
 
 func (p *sessionPool) decNumInUseLocked(ctx context.Context) {
 	p.numInUse--
+	if int64(p.numInUse) < 0 {
+		// print whole call stack trace
+		logf(p.sc.logger, "Number of sessions in use is negative, resetting it to currSessionsCheckedOutLocked. Stack trace: %s", string(debug.Stack()))
+		p.numInUse = p.currSessionsCheckedOutLocked()
+	}
 	p.recordStat(ctx, SessionsCount, int64(p.numInUse), tagNumInUseSessions)
 	p.recordStat(ctx, ReleasedSessionsCount, 1)
 	if p.otConfig != nil {
@@ -1459,12 +1464,12 @@ func (hc *healthChecker) healthCheck(s *session) {
 	defer hc.markDone(s)
 	if !s.pool.isValid() {
 		// Session pool is closed, perform a garbage collection.
-		s.destroy(false, true)
+		s.destroy(false, false)
 		return
 	}
 	if err := s.ping(); isSessionNotFoundError(err) {
 		// Ping failed, destroy the session.
-		s.destroy(false, true)
+		s.destroy(false, false)
 	}
 }