Skip to content

Commit

Permalink
Avoid reading MemStats too often
Browse files Browse the repository at this point in the history 
Signed-off-by: Bogdan Drutu <bogdandrutu@gmail.com>
  • Loading branch information
@bogdandrutu
bogdandrutu committed Jun 8, 2022
1 parent ea21df4 commit f560f76
Showing 1 changed file with 50 additions and 45 deletions.
95 changes: 50 additions & 45 deletions service/internal/telemetry/process_telemetry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,138 +36,143 @@ type ProcessMetrics struct {
cpuSeconds *metric.Float64DerivedCumulative
rssMemory *metric.Int64DerivedGauge

proc *process.Process
lastMsRead time.Time
ms *runtime.MemStats
proc *process.Process
}

// RegisterProcessMetrics creates a new set of ProcessMetrics (mem, cpu) that can be used to measure
// basic information about this process.
func RegisterProcessMetrics(registry *metric.Registry, ballastSizeBytes uint64) error {
pmv := &ProcessMetrics{
pm := &ProcessMetrics{
startTimeUnixNano: time.Now().UnixNano(),
ballastSizeBytes: ballastSizeBytes,
ms: &runtime.MemStats{},
}

var err error
pmv.processUptime, err = registry.AddFloat64DerivedCumulative(
pm.proc, err = process.NewProcess(int32(os.Getpid()))
if err != nil {
return err
}

pm.processUptime, err = registry.AddFloat64DerivedCumulative(
"process/uptime",
metric.WithDescription("Uptime of the process"),
metric.WithUnit(stats.UnitSeconds))
if err != nil {
return err
}
if err = pmv.processUptime.UpsertEntry(pmv.updateProcessUptime); err != nil {
if err = pm.processUptime.UpsertEntry(pm.updateProcessUptime); err != nil {
return err
}

pmv.allocMem, err = registry.AddInt64DerivedGauge(
pm.allocMem, err = registry.AddInt64DerivedGauge(
"process/runtime/heap_alloc_bytes",
metric.WithDescription("Bytes of allocated heap objects (see 'go doc runtime.MemStats.HeapAlloc')"),
metric.WithUnit(stats.UnitBytes))
if err != nil {
return err
}
if err = pmv.allocMem.UpsertEntry(pmv.updateAllocMem); err != nil {
if err = pm.allocMem.UpsertEntry(pm.updateAllocMem); err != nil {
return err
}

pmv.totalAllocMem, err = registry.AddInt64DerivedCumulative(
pm.totalAllocMem, err = registry.AddInt64DerivedCumulative(
"process/runtime/total_alloc_bytes",
metric.WithDescription("Cumulative bytes allocated for heap objects (see 'go doc runtime.MemStats.TotalAlloc')"),
metric.WithUnit(stats.UnitBytes))
if err != nil {
return err
}
if err = pmv.totalAllocMem.UpsertEntry(pmv.updateTotalAllocMem); err != nil {
if err = pm.totalAllocMem.UpsertEntry(pm.updateTotalAllocMem); err != nil {
return err
}

pmv.sysMem, err = registry.AddInt64DerivedGauge(
pm.sysMem, err = registry.AddInt64DerivedGauge(
"process/runtime/total_sys_memory_bytes",
metric.WithDescription("Total bytes of memory obtained from the OS (see 'go doc runtime.MemStats.Sys')"),
metric.WithUnit(stats.UnitBytes))
if err != nil {
return err
}
if err = pmv.sysMem.UpsertEntry(pmv.updateSysMem); err != nil {
if err = pm.sysMem.UpsertEntry(pm.updateSysMem); err != nil {
return err
}

pmv.cpuSeconds, err = registry.AddFloat64DerivedCumulative(
pm.cpuSeconds, err = registry.AddFloat64DerivedCumulative(
"process/cpu_seconds",
metric.WithDescription("Total CPU user and system time in seconds"),
metric.WithUnit(stats.UnitSeconds))
if err != nil {
return err
}
if err = pmv.cpuSeconds.UpsertEntry(pmv.updateCPUSeconds); err != nil {
if err = pm.cpuSeconds.UpsertEntry(pm.updateCPUSeconds); err != nil {
return err
}

pmv.rssMemory, err = registry.AddInt64DerivedGauge(
pm.rssMemory, err = registry.AddInt64DerivedGauge(
"process/memory/rss",
metric.WithDescription("Total physical memory (resident set size)"),
metric.WithUnit(stats.UnitBytes))
if err != nil {
return err
}
if err = pmv.rssMemory.UpsertEntry(pmv.updateRSSMemory); err != nil {
return err
}

pmv.proc, err = process.NewProcess(int32(os.Getpid()))
if err != nil {
if err = pm.rssMemory.UpsertEntry(pm.updateRSSMemory); err != nil {
return err
}

return nil
}

func (pmv *ProcessMetrics) updateProcessUptime() float64 {
func (pm *ProcessMetrics) updateProcessUptime() float64 {
now := time.Now().UnixNano()
return float64(now-pmv.startTimeUnixNano) / 1e9
return float64(now-pm.startTimeUnixNano) / 1e9
}

func (pmv *ProcessMetrics) updateAllocMem() int64 {
ms := runtime.MemStats{}
pmv.readMemStats(&ms)
return int64(ms.Alloc)
func (pm *ProcessMetrics) updateAllocMem() int64 {
pm.readMemStatsIfNeeded()
return int64(pm.ms.Alloc)
}

func (pmv *ProcessMetrics) updateTotalAllocMem() int64 {
ms := runtime.MemStats{}
pmv.readMemStats(&ms)
return int64(ms.TotalAlloc)
func (pm *ProcessMetrics) updateTotalAllocMem() int64 {
pm.readMemStatsIfNeeded()
return int64(pm.ms.TotalAlloc)
}

func (pmv *ProcessMetrics) updateSysMem() int64 {
ms := runtime.MemStats{}
pmv.readMemStats(&ms)
return int64(ms.Sys)
func (pm *ProcessMetrics) updateSysMem() int64 {
pm.readMemStatsIfNeeded()
return int64(pm.ms.Sys)
}

func (pmv *ProcessMetrics) updateCPUSeconds() float64 {
times, err := pmv.proc.Times()
func (pm *ProcessMetrics) updateCPUSeconds() float64 {
times, err := pm.proc.Times()
if err != nil {
return 0
}

return times.Total()
}

func (pmv *ProcessMetrics) updateRSSMemory() int64 {
mem, err := pmv.proc.MemoryInfo()
func (pm *ProcessMetrics) updateRSSMemory() int64 {
mem, err := pm.proc.MemoryInfo()
if err != nil {
return 0
}
return int64(mem.RSS)
}

func (pmv *ProcessMetrics) readMemStats(ms *runtime.MemStats) {
runtime.ReadMemStats(ms)
if pmv.ballastSizeBytes > 0 {
ms.Alloc -= pmv.ballastSizeBytes
ms.HeapAlloc -= pmv.ballastSizeBytes
ms.HeapSys -= pmv.ballastSizeBytes
ms.HeapInuse -= pmv.ballastSizeBytes
func (pm *ProcessMetrics) readMemStatsIfNeeded() {
now := time.Now()
// If last time we read was less than one second ago just reuse the values
if now.Sub(pm.lastMsRead) < time.Second {
return
}
pm.lastMsRead = now
runtime.ReadMemStats(pm.ms)
if pm.ballastSizeBytes > 0 {
pm.ms.Alloc -= pm.ballastSizeBytes
pm.ms.HeapAlloc -= pm.ballastSizeBytes
pm.ms.HeapSys -= pm.ballastSizeBytes
pm.ms.HeapInuse -= pm.ballastSizeBytes
}
}

0 comments on commit f560f76

Please sign in to comment.