Add gauges for all monitors in content node

creator-node/src/components/healthCheck/healthCheckController.js

@@ -61,13 +61,6 @@ const healthCheckController = async (req) => {
     randomBytesToSign
   )
 
-  const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry
-  const storagePathSizeMetric = prometheusRegistry.getMetric(
-    prometheusRegistry.metricNames.STORAGE_PATH_SIZE_BYTES
-  )
-  storagePathSizeMetric.set({ type: 'total' }, response.storagePathSize)
-  storagePathSizeMetric.set({ type: 'used' }, response.storagePathUsed)
-
   if (enforceStateMachineQueueHealth) {
     const { stateMachineJobs } = response
     const {

creator-node/src/monitors/MonitoringQueue.js

@@ -1,7 +1,11 @@
 const Bull = require('bull')
 const redis = require('../redis')
 const config = require('../config')
-const { MONITORS, getMonitorRedisKey } = require('./monitors')
+const {
+  MONITORS,
+  PROMETHEUS_MONITORS,
+  getMonitorRedisKey
+} = require('./monitors')
 const { logger } = require('../logging')
 
 const QUEUE_INTERVAL_MS = 60 * 1000
@@ -22,7 +26,7 @@ const MONITORING_QUEUE_HISTORY = 500
  *  2. Refreshes the value and stores the update in redis
  */
 class MonitoringQueue {
-  constructor() {
+  constructor(prometheusRegistry) {
     this.queue = new Bull('monitoring-queue', {
       redis: {
         port: config.get('redisPort'),
@@ -34,6 +38,8 @@ class MonitoringQueue {
       }
     })
 
+    this.prometheusRegistry = prometheusRegistry
+
     // Clean up anything that might be still stuck in the queue on restart
     this.queue.empty()
 
@@ -48,11 +54,11 @@ class MonitoringQueue {
 
           // Iterate over each monitor and set a new value if the cached
           // value is not fresh.
-          Object.values(MONITORS).forEach(async (monitor) => {
+          Object.keys(MONITORS).forEach(async (monitorKey) => {
             try {
-              await this.refresh(monitor)
+              await this.refresh(MONITORS[monitorKey], monitorKey)
             } catch (e) {
-              this.logStatus(`Error on ${monitor.name} ${e}`)
+              this.logStatus(`Error on ${monitorKey} ${e}`)
             }
           })
 
@@ -71,29 +77,48 @@ class MonitoringQueue {
    * them on init
    */
   async seedInitialValues() {
-    await this.refresh(MONITORS.STORAGE_PATH_SIZE)
-    await this.refresh(MONITORS.STORAGE_PATH_USED)
+    await this.refresh(MONITORS.STORAGE_PATH_SIZE, 'STORAGE_PATH_SIZE')
+    await this.refresh(MONITORS.STORAGE_PATH_USED, 'STORAGE_PATH_USED')
   }
 
-  async refresh(monitor) {
-    const key = getMonitorRedisKey(monitor)
+  /**
+   * Refresh monitor in redis and prometheus (if integer)
+   * @param {Object} monitorVal Object containing the monitor props like { func, ttl, type, name }
+   * @param {*} monitorKey name of the monitor eg `THIRTY_DAY_ROLLING_SYNC_SUCCESS_COUNT`
+   */
+  async refresh(monitorVal, monitorKey) {
+    const key = getMonitorRedisKey(monitorVal)
     const ttlKey = `${key}:ttl`
 
     // If the value is fresh, exit early
     const isFresh = await redis.get(ttlKey)
     if (isFresh) return
 
-    const value = await monitor.func()
-    this.logStatus(`Computed value for ${monitor.name} ${value}`)
+    const value = await monitorVal.func()
+    this.logStatus(`Computed value for ${monitorVal.name} ${value}`)
+
+    // store integer monitors in prometheus
+    try {
+      if (PROMETHEUS_MONITORS.hasOwnProperty(monitorKey)) {
+        const metric = this.prometheusRegistry.getMetric(
+          this.prometheusRegistry.metricNames[`MONITOR_${monitorKey}`]
+        )
+        metric.set({}, value)
+      }
+    } catch (e) {
+      logger.debug(
+        `Couldn't store value: ${value} in prometheus for metric: ${monitorKey}`
+      )
+    }
 
     // Set the value
     redis.set(key, value)
 
-    if (monitor.ttl) {
+    if (monitorVal.ttl) {
       // Set a TTL (in seconds) key to track when this value needs refreshing.
       // We store a separate TTL key rather than expiring the value itself
       // so that in the case of an error, the current value can still be read
-      redis.set(ttlKey, 1, 'EX', monitor.ttl)
+      redis.set(ttlKey, 1, 'EX', monitorVal.ttl)
     }
   }
 

creator-node/src/monitors/monitors.js

@@ -294,6 +294,12 @@ const MONITORS = {
   LATEST_FIND_REPLICA_SET_UPDATES_JOB_SUCCESS
 }
 
+const PROMETHEUS_MONITORS = {}
+for (const monitor in MONITORS) {
+  if (MONITORS[monitor]?.type === 'int')
+    PROMETHEUS_MONITORS[monitor] = MONITORS[monitor]
+}
+
 const getMonitorRedisKey = (monitor) =>
   `${MONITORING_REDIS_PREFIX}:${monitor.name}`
 
@@ -346,6 +352,7 @@ const getMonitors = async (monitors) => {
 
 module.exports = {
   MONITORS,
+  PROMETHEUS_MONITORS,
   getMonitorRedisKey,
   getMonitors
 }

creator-node/src/serviceRegistry.js

@@ -49,7 +49,7 @@ class ServiceRegistry {
     this.trustedNotifierManager = null // Service that blacklists content on behalf of Content Nodes
 
     // Queues
-    this.monitoringQueue = new MonitoringQueue() // Recurring job to monitor node state & performance metrics
+    this.monitoringQueue = new MonitoringQueue(this.prometheusRegistry) // Recurring job to monitor node state & performance metrics
     this.sessionExpirationQueue = new SessionExpirationQueue() // Recurring job to clear expired session tokens from Redis and DB
     this.imageProcessingQueue = new ImageProcessingQueue() // Resizes all images on Audius
     this.transcodingQueue = TranscodingQueue // Transcodes and segments all tracks

creator-node/src/services/prometheusMonitoring/prometheus.constants.ts

@@ -10,6 +10,7 @@ import {
   // eslint-disable-next-line import/no-unresolved
 } from '../stateMachineManager/stateMachineConstants'
 import * as config from '../../config'
+import { PROMETHEUS_MONITORS } from '../../monitors/monitors'
 
 /**
  * For explanation of METRICS, and instructions on how to add a new metric, please see `prometheusMonitoring/README.md`
@@ -58,8 +59,7 @@ const metricNames: Record<string, string> = {
   JOBS_ATTEMPTS_HISTOGRAM: 'jobs_attempts',
   RECOVER_ORPHANED_DATA_WALLET_COUNTS_GAUGE:
     'recover_orphaned_data_wallet_counts',
-  RECOVER_ORPHANED_DATA_SYNC_COUNTS_GAUGE: 'recover_orphaned_data_sync_counts',
-  STORAGE_PATH_SIZE_BYTES: 'storage_path_size_bytes'
+  RECOVER_ORPHANED_DATA_SYNC_COUNTS_GAUGE: 'recover_orphaned_data_sync_counts'
 }
 // Add a histogram for each job in the state machine queues.
 // Some have custom labels below, and all of them use the label: uncaughtError=true/false
@@ -70,6 +70,11 @@ for (const jobName of Object.values(
     `STATE_MACHINE_${jobName}_JOB_DURATION_SECONDS_HISTOGRAM`
   ] = `state_machine_${snakeCase(jobName)}_job_duration_seconds`
 }
+// Add gauge for each monitor
+for (const monitor of Object.keys(PROMETHEUS_MONITORS)) {
+  metricNames[`MONITOR_${monitor}`] = `monitor_${snakeCase(monitor)}`
+}
+
 export const METRIC_NAMES = Object.freeze(
   mapValues(metricNames, (metricName) => `${NAMESPACE_PREFIX}_${metricName}`)
 )
@@ -348,6 +353,22 @@ export const METRICS: Record<string, Metric> = Object.freeze({
       }
     ])
   ),
+  // Add gauge for each monitor
+  ...Object.fromEntries(
+    Object.keys(PROMETHEUS_MONITORS).map((monitor) => {
+      return [
+        METRIC_NAMES[`MONITOR_${monitor}`],
+        {
+          metricType: METRIC_TYPES.GAUGE,
+          metricConfig: {
+            name: METRIC_NAMES[`MONITOR_${monitor}`],
+            help: `Record monitor: ${monitor}`,
+            labelNames: []
+          }
+        }
+      ]
+    })
+  ),
   [METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE]: {
     metricType: METRIC_TYPES.GAUGE,
     metricConfig: {
@@ -374,14 +395,6 @@ export const METRICS: Record<string, Metric> = Object.freeze({
         5
       )
     }
-  },
-  [METRIC_NAMES.STORAGE_PATH_SIZE_BYTES]: {
-    metricType: METRIC_TYPES.GAUGE,
-    metricConfig: {
-      name: METRIC_NAMES.STORAGE_PATH_SIZE_BYTES,
-      help: 'Disk storage size',
-      labelNames: ['type']
-    }
   }
 })
 

creator-node/test/lib/app.js

@@ -32,7 +32,7 @@ async function getApp(
     libs: libsClient,
     blacklistManager,
     redis: redisClient,
-    monitoringQueue: new MonitoringQueueMock(),
+    monitoringQueue: new MonitoringQueueMock(prometheusRegistry),
     asyncProcessingQueue: apq,
     imageProcessingQueue: new ImageProcessingQueue(),
     nodeConfig,
@@ -55,15 +55,16 @@ async function getApp(
 }
 
 function getServiceRegistryMock(libsClient, blacklistManager) {
+  const prometheusRegistry = new PrometheusRegistry()
   return {
     libs: libsClient,
     blacklistManager: blacklistManager,
     redis: redisClient,
-    monitoringQueue: new MonitoringQueueMock(),
+    monitoringQueue: new MonitoringQueueMock(prometheusRegistry),
     syncQueue: new SyncQueue(nodeConfig, redisClient),
     nodeConfig,
     initLibs: async function () {},
-    prometheusRegistry: new PrometheusRegistry()
+    prometheusRegistry
   }
 }
 

creator-node/test/lib/monitoringQueueMock.js

@@ -3,7 +3,8 @@ const redisClient = require('../../src/redis')
 
 // Mock monitoring queue that sets monitor values on construction
 class MonitoringQueueMock {
-  constructor () {
+  constructor (prometheusRegistry) {
+    this.prometheusRegistry = prometheusRegistry
     redisClient.set(
       getMonitorRedisKey(MONITORS.DATABASE_LIVENESS),
       'true'