diff --git a/service/internal/telemetry/process_telemetry.go b/service/internal/telemetry/process_telemetry.go index 435c5af97d5..e1a43cb1eb2 100644 --- a/service/internal/telemetry/process_telemetry.go +++ b/service/internal/telemetry/process_telemetry.go @@ -36,117 +36,116 @@ type ProcessMetrics struct { cpuSeconds *metric.Float64DerivedCumulative rssMemory *metric.Int64DerivedGauge - proc *process.Process + lastMsRead time.Time + ms *runtime.MemStats + proc *process.Process } // RegisterProcessMetrics creates a new set of ProcessMetrics (mem, cpu) that can be used to measure // basic information about this process. func RegisterProcessMetrics(registry *metric.Registry, ballastSizeBytes uint64) error { - pmv := &ProcessMetrics{ + pm := &ProcessMetrics{ startTimeUnixNano: time.Now().UnixNano(), ballastSizeBytes: ballastSizeBytes, + ms: &runtime.MemStats{}, } - var err error - pmv.processUptime, err = registry.AddFloat64DerivedCumulative( + pm.proc, err = process.NewProcess(int32(os.Getpid())) + if err != nil { + return err + } + + pm.processUptime, err = registry.AddFloat64DerivedCumulative( "process/uptime", metric.WithDescription("Uptime of the process"), metric.WithUnit(stats.UnitSeconds)) if err != nil { return err } - if err = pmv.processUptime.UpsertEntry(pmv.updateProcessUptime); err != nil { + if err = pm.processUptime.UpsertEntry(pm.updateProcessUptime); err != nil { return err } - pmv.allocMem, err = registry.AddInt64DerivedGauge( + pm.allocMem, err = registry.AddInt64DerivedGauge( "process/runtime/heap_alloc_bytes", metric.WithDescription("Bytes of allocated heap objects (see 'go doc runtime.MemStats.HeapAlloc')"), metric.WithUnit(stats.UnitBytes)) if err != nil { return err } - if err = pmv.allocMem.UpsertEntry(pmv.updateAllocMem); err != nil { + if err = pm.allocMem.UpsertEntry(pm.updateAllocMem); err != nil { return err } - pmv.totalAllocMem, err = registry.AddInt64DerivedCumulative( + pm.totalAllocMem, err = registry.AddInt64DerivedCumulative( "process/runtime/total_alloc_bytes", metric.WithDescription("Cumulative bytes allocated for heap objects (see 'go doc runtime.MemStats.TotalAlloc')"), metric.WithUnit(stats.UnitBytes)) if err != nil { return err } - if err = pmv.totalAllocMem.UpsertEntry(pmv.updateTotalAllocMem); err != nil { + if err = pm.totalAllocMem.UpsertEntry(pm.updateTotalAllocMem); err != nil { return err } - pmv.sysMem, err = registry.AddInt64DerivedGauge( + pm.sysMem, err = registry.AddInt64DerivedGauge( "process/runtime/total_sys_memory_bytes", metric.WithDescription("Total bytes of memory obtained from the OS (see 'go doc runtime.MemStats.Sys')"), metric.WithUnit(stats.UnitBytes)) if err != nil { return err } - if err = pmv.sysMem.UpsertEntry(pmv.updateSysMem); err != nil { + if err = pm.sysMem.UpsertEntry(pm.updateSysMem); err != nil { return err } - pmv.cpuSeconds, err = registry.AddFloat64DerivedCumulative( + pm.cpuSeconds, err = registry.AddFloat64DerivedCumulative( "process/cpu_seconds", metric.WithDescription("Total CPU user and system time in seconds"), metric.WithUnit(stats.UnitSeconds)) if err != nil { return err } - if err = pmv.cpuSeconds.UpsertEntry(pmv.updateCPUSeconds); err != nil { + if err = pm.cpuSeconds.UpsertEntry(pm.updateCPUSeconds); err != nil { return err } - pmv.rssMemory, err = registry.AddInt64DerivedGauge( + pm.rssMemory, err = registry.AddInt64DerivedGauge( "process/memory/rss", metric.WithDescription("Total physical memory (resident set size)"), metric.WithUnit(stats.UnitBytes)) if err != nil { return err } - if err = pmv.rssMemory.UpsertEntry(pmv.updateRSSMemory); err != nil { - return err - } - - pmv.proc, err = process.NewProcess(int32(os.Getpid())) - if err != nil { + if err = pm.rssMemory.UpsertEntry(pm.updateRSSMemory); err != nil { return err } return nil } -func (pmv *ProcessMetrics) updateProcessUptime() float64 { +func (pm *ProcessMetrics) updateProcessUptime() float64 { now := time.Now().UnixNano() - return float64(now-pmv.startTimeUnixNano) / 1e9 + return float64(now-pm.startTimeUnixNano) / 1e9 } -func (pmv *ProcessMetrics) updateAllocMem() int64 { - ms := runtime.MemStats{} - pmv.readMemStats(&ms) - return int64(ms.Alloc) +func (pm *ProcessMetrics) updateAllocMem() int64 { + pm.readMemStatsIfNeeded() + return int64(pm.ms.Alloc) } -func (pmv *ProcessMetrics) updateTotalAllocMem() int64 { - ms := runtime.MemStats{} - pmv.readMemStats(&ms) - return int64(ms.TotalAlloc) +func (pm *ProcessMetrics) updateTotalAllocMem() int64 { + pm.readMemStatsIfNeeded() + return int64(pm.ms.TotalAlloc) } -func (pmv *ProcessMetrics) updateSysMem() int64 { - ms := runtime.MemStats{} - pmv.readMemStats(&ms) - return int64(ms.Sys) +func (pm *ProcessMetrics) updateSysMem() int64 { + pm.readMemStatsIfNeeded() + return int64(pm.ms.Sys) } -func (pmv *ProcessMetrics) updateCPUSeconds() float64 { - times, err := pmv.proc.Times() +func (pm *ProcessMetrics) updateCPUSeconds() float64 { + times, err := pm.proc.Times() if err != nil { return 0 } @@ -154,20 +153,26 @@ func (pmv *ProcessMetrics) updateCPUSeconds() float64 { return times.Total() } -func (pmv *ProcessMetrics) updateRSSMemory() int64 { - mem, err := pmv.proc.MemoryInfo() +func (pm *ProcessMetrics) updateRSSMemory() int64 { + mem, err := pm.proc.MemoryInfo() if err != nil { return 0 } return int64(mem.RSS) } -func (pmv *ProcessMetrics) readMemStats(ms *runtime.MemStats) { - runtime.ReadMemStats(ms) - if pmv.ballastSizeBytes > 0 { - ms.Alloc -= pmv.ballastSizeBytes - ms.HeapAlloc -= pmv.ballastSizeBytes - ms.HeapSys -= pmv.ballastSizeBytes - ms.HeapInuse -= pmv.ballastSizeBytes +func (pm *ProcessMetrics) readMemStatsIfNeeded() { + now := time.Now() + // If last time we read was less than one second ago just reuse the values + if now.Sub(pm.lastMsRead) < time.Second { + return + } + pm.lastMsRead = now + runtime.ReadMemStats(pm.ms) + if pm.ballastSizeBytes > 0 { + pm.ms.Alloc -= pm.ballastSizeBytes + pm.ms.HeapAlloc -= pm.ballastSizeBytes + pm.ms.HeapSys -= pm.ballastSizeBytes + pm.ms.HeapInuse -= pm.ballastSizeBytes } }