diff --git a/creator-node/package-lock.json b/creator-node/package-lock.json index d197abf740d..4651cf5f76e 100644 --- a/creator-node/package-lock.json +++ b/creator-node/package-lock.json @@ -6904,6 +6904,15 @@ } } }, + "express-prom-bundle": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/express-prom-bundle/-/express-prom-bundle-6.5.0.tgz", + "integrity": "sha512-paFAm0FK7TV1Ln6Blh9edDt2mJ4Pk6Py/fjhZDMcoMHENYryBjCpnXDXuCu8NE1kkvp58IrPcAAkNeNqdvZnnw==", + "requires": { + "on-finished": "^2.3.0", + "url-value-parser": "^2.0.0" + } + }, "express-rate-limit": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.3.0.tgz", @@ -14381,6 +14390,11 @@ "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k=" }, + "url-value-parser": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/url-value-parser/-/url-value-parser-2.1.0.tgz", + "integrity": "sha512-gIYPWXujdUdwd/9TGCHTf5Vvgw6lOxjE5Q/k+7WNByYyS0vW5WX0k+xuVlhvPq6gRNhzXVv/ezC+OfeAet5Kcw==" + }, "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", diff --git a/creator-node/package.json b/creator-node/package.json index 6ccace2ceb7..6c54bd6a43e 100644 --- a/creator-node/package.json +++ b/creator-node/package.json @@ -47,6 +47,7 @@ "ethers": "5.4.7", "exif-parser": "^0.1.12", "express": "^4.16.3", + "express-prom-bundle": "^6.5.0", "express-rate-limit": "5.3.0", "ffmpeg-static": "^2.7.0", "ffprobe-static": "^3.0.0", diff --git a/creator-node/src/app.js b/creator-node/src/app.js index 4b671562b46..ff95d98250f 100644 --- a/creator-node/src/app.js +++ b/creator-node/src/app.js @@ -1,6 +1,7 @@ const express = require('express') const bodyParser = require('body-parser') const cors = require('cors') +const prometheusMiddleware = require('express-prom-bundle') const DiskManager = require('./diskManager') const { sendResponse, errorResponseServerError } = require('./apiHelpers') @@ -19,42 +20,18 @@ const { getRateLimiterMiddleware } = require('./reqLimiter') const config = require('./config') +const { + exponentialBucketsRange +} = require('./services/prometheusMonitoring/prometheusUtils') const healthCheckRoutes = require('./components/healthCheck/healthCheckController') const contentBlacklistRoutes = require('./components/contentBlacklist/contentBlacklistController') const replicaSetRoutes = require('./components/replicaSet/replicaSetController') -const app = express() - -// middleware functions will be run in order they are added to the app below -// - loggingMiddleware must be first to ensure proper error handling -app.use(loggingMiddleware) -app.use(bodyParser.json({ limit: '1mb' })) -app.use(readOnlyMiddleware) -app.use(cors()) - -// Rate limit routes -app.use('/users/', userReqLimiter) -app.use('/track*', trackReqLimiter) -app.use('/audius_user/', audiusUserReqLimiter) -app.use('/metadata', metadataReqLimiter) -app.use('/image_upload', imageReqLimiter) -app.use('/ursm_request_for_signature', URSMRequestForSignatureReqLimiter) -app.use('/batch_cids_exist', batchCidsExistReqLimiter) -app.use('/batch_image_cids_exist', batchCidsExistReqLimiter) -app.use(getRateLimiterMiddleware()) - -// import routes -require('./routes')(app) -app.use('/', healthCheckRoutes) -app.use('/', contentBlacklistRoutes) -app.use('/', replicaSetRoutes) - function errorHandler(err, req, res, next) { req.logger.error('Internal server error') req.logger.error(err.stack) sendResponse(req, res, errorResponseServerError('Internal server error')) } -app.use(errorHandler) /** * Configures express app object with required properties and starts express server @@ -63,6 +40,78 @@ app.use(errorHandler) * @param {ServiceRegistry} serviceRegistry object housing all Content Node Services */ const initializeApp = (port, serviceRegistry) => { + const app = express() + + // middleware functions will be run in order they are added to the app below + // - loggingMiddleware must be first to ensure proper error handling + app.use(loggingMiddleware) + app.use(bodyParser.json({ limit: '1mb' })) + app.use(readOnlyMiddleware) + app.use(cors()) + + // Rate limit routes + app.use('/users/', userReqLimiter) + app.use('/track*', trackReqLimiter) + app.use('/audius_user/', audiusUserReqLimiter) + app.use('/metadata', metadataReqLimiter) + app.use('/image_upload', imageReqLimiter) + app.use('/ursm_request_for_signature', URSMRequestForSignatureReqLimiter) + app.use('/batch_cids_exist', batchCidsExistReqLimiter) + app.use('/batch_image_cids_exist', batchCidsExistReqLimiter) + app.use(getRateLimiterMiddleware()) + + const prometheusRegistry = serviceRegistry.prometheusRegistry + + // Metric tracking middleware + app.use( + prometheusMiddleware({ + // Use existing registry for compatibility with custom metrics. Can see + // the metrics on /prometheus_metrics + promRegistry: prometheusRegistry.registry, + // Override metric name to include namespace prefix + httpDurationMetricName: `${prometheusRegistry.namespacePrefix}_http_request_duration_seconds`, + // Include HTTP method in duration tracking + includeMethod: true, + // Include HTTP status code in duration tracking + includePath: true, + // Disable default gauge counter to indicate if this middleware is running + includeUp: false, + // The buckets in seconds to measure requests + buckets: [0.2, 0.5, ...exponentialBucketsRange(1, 60, 4)], + // Do not register the default /metrics route, since we have the /prometheus_metrics + autoregister: false, + // Normalizes the path to be tracked in this middleware. For routes with route params, + // this fn maps those routes to generic paths. e.g. /ipfs/QmSomeCid -> /ipfs/#CID + normalizePath: function (req, opts) { + const path = prometheusMiddleware.normalizePath(req, opts) + try { + for (const { + regex, + path: normalizedPath + } of prometheusRegistry.regexes) { + const match = path.match(regex) + if (match) { + return normalizedPath + } + } + } catch (e) { + req.logger.warn( + `DurationTracking || Could not match on regex: ${e.message}` + ) + } + return path + } + }) + ) + + // import routes + require('./routes')(app) + app.use('/', healthCheckRoutes) + app.use('/', contentBlacklistRoutes) + app.use('/', replicaSetRoutes) + + app.use(errorHandler) + const storagePath = DiskManager.getConfigStoragePath() // TODO: Can remove these when all routes consume serviceRegistry diff --git a/creator-node/src/routes/tracks.js b/creator-node/src/routes/tracks.js index 7ba48248b0e..f2938492392 100644 --- a/creator-node/src/routes/tracks.js +++ b/creator-node/src/routes/tracks.js @@ -321,17 +321,8 @@ module.exports = function (app) { transcodedTrackUUID } = req.body - const prometheusRegistry = - req.app.get('serviceRegistry').prometheusRegistry - const routePostTracksDurationSecondsMetric = prometheusRegistry.getMetric( - prometheusRegistry.metricNames - .ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM - ) - const metricEndTimerFn = routePostTracksDurationSecondsMetric.startTimer() - // Input validation if (!blockchainTrackId || !blockNumber || !metadataFileUUID) { - metricEndTimerFn({ code: 400 }) return errorResponseBadRequest( 'Must include blockchainTrackId, blockNumber, and metadataFileUUID.' ) @@ -340,7 +331,6 @@ module.exports = function (app) { // Error on outdated blocknumber const cnodeUser = req.session.cnodeUser if (blockNumber < cnodeUser.latestBlockNumber) { - metricEndTimerFn({ code: 400 }) return errorResponseBadRequest( `Invalid blockNumber param ${blockNumber}. Must be greater or equal to previously processed blocknumber ${cnodeUser.latestBlockNumber}.` ) @@ -352,7 +342,6 @@ module.exports = function (app) { where: { fileUUID: metadataFileUUID, cnodeUserUUID } }) if (!file) { - metricEndTimerFn({ code: 400 }) return errorResponseBadRequest( `No file db record found for provided metadataFileUUID ${metadataFileUUID}.` ) @@ -367,13 +356,11 @@ module.exports = function (app) { !Array.isArray(metadataJSON.track_segments) || !metadataJSON.track_segments.length ) { - metricEndTimerFn({ code: 500 }) return errorResponseServerError( `Malformatted metadataJSON stored for metadataFileUUID ${metadataFileUUID}.` ) } } catch (e) { - metricEndTimerFn({ code: 500 }) return errorResponseServerError( `No file stored on disk for metadataFileUUID ${metadataFileUUID} at storagePath ${file.storagePath}.` ) @@ -387,7 +374,6 @@ module.exports = function (app) { metadataJSON.cover_art_sizes ) } catch (e) { - metricEndTimerFn({ code: 500 }) return errorResponseServerError(e.message) } @@ -575,12 +561,10 @@ module.exports = function (app) { // Discovery only indexes metadata and not files, so we eagerly replicate data but don't await it issueAndWaitForSecondarySyncRequests(req, true) - metricEndTimerFn({ code: 200 }) return successResponse() } catch (e) { req.logger.error(e.message) await transaction.rollback() - metricEndTimerFn({ code: 500 }) return errorResponseServerError(e.message) } }) diff --git a/creator-node/src/serviceRegistry.js b/creator-node/src/serviceRegistry.js index ff219219595..9974b8e19c2 100644 --- a/creator-node/src/serviceRegistry.js +++ b/creator-node/src/serviceRegistry.js @@ -317,6 +317,14 @@ class ServiceRegistry { ) } + try { + await this._setupRouteDurationTracking(app) + } catch (e) { + this.logError( + `DurationTracking || Failed to setup general duration tracking for all routes: ${e.message}. Skipping..` + ) + } + this.servicesThatRequireServerInitialized = true logInfoWithDuration( @@ -368,6 +376,95 @@ class ServiceRegistry { ) } + /** + * Gets the regexes for routes with route params. Used for mapping paths in metrics to track route durations + * + * Structure: + * {regex: , path: } + * + * Example of the regex added to PrometheusRegistry: + * + * /ipfs/:CID and /content/:CID -> map to /ipfs/#CID + * + * { + * regex: /(?:^\/ipfs\/(?:([^/]+?))\/?$|^\/content\/(?:([^/]+?))\/?$)/i, + * path: '/ipfs/#CID' + * } + * @param {Object} app + */ + async _setupRouteDurationTracking(app) { + // Get all the routes initialized in the app + const routes = app._router.stack.filter( + (element) => + (element.route && element.route.path) || + (element.handle && element.handle.stack) + ) + + // Iterate through the paths and add the regexes if the route + // has route params + + // Express routing is... unpredictable. Use a set to manage seen paths + const seenPaths = new Set() + const parsedRoutes = [] + routes.forEach((element) => { + if (element.name === 'router') { + // Iterate through routes initialized with the express router + element.handle.stack.forEach((e) => { + const path = e.route.path + const method = Object.keys(e.route.methods)[0] + const regex = e.regexp + addToParsedRoutes(path, method, regex) + }) + } else { + // Iterate through all the other routes initalized with the app + const path = element.route.path + const method = Object.keys(element.route.methods)[0] + const regex = element.regexp + addToParsedRoutes(path, method, regex) + } + }) + + // Only keep track of the routes with route params, e.g. the route with ':' + // in the route to indicate a route param + function addToParsedRoutes(path, method, regex) { + // Routes may come in the form of an array (e.g. ['/ipfs/:CID', '/content/:CID']) + if (Array.isArray(path)) { + path.forEach((p) => { + if (!seenPaths.has(p + method)) { + if (p.includes(':')) { + seenPaths.add(p + method) + parsedRoutes.push({ + path: p, + method, + regex + }) + } + } + }) + } else { + if (!seenPaths.has(path + method)) { + if (path.includes(':')) { + seenPaths.add(path + method) + parsedRoutes.push({ + path, + method, + regex + }) + } + } + } + } + + const regexes = parsedRoutes + .filter(({ path }) => path.includes(':')) + .map(({ path, regex }) => ({ + regex, + path: path.replace(/:/g, '#') + })) + + this.prometheusRegistry.initRouteParamRegexes(regexes) + } + /** * Wait until URSM contract is deployed, then attempt to register on L2 URSM with infinite retries * Requires L1 identity @@ -476,11 +573,7 @@ class ServiceRegistry { } } -/* - * Export a singleton instance of the ServiceRegistry - */ -const serviceRegistry = new ServiceRegistry() - +// Export a singleton instance of the ServiceRegistry module.exports = { - serviceRegistry + serviceRegistry: new ServiceRegistry() } diff --git a/creator-node/src/services/prometheusMonitoring/README.md b/creator-node/src/services/prometheusMonitoring/README.md index cbfe95a9be3..2d4e84b3592 100644 --- a/creator-node/src/services/prometheusMonitoring/README.md +++ b/creator-node/src/services/prometheusMonitoring/README.md @@ -97,11 +97,11 @@ Given the above, some important conclusions: ### Create metric in `./prometheus.constants.js` -Add row to `MetricNames` +Add row to `METRIC_NAMES` - Key should include a suffix for metric type (either _GAUGE or _HISTOGRAM) -Add new object to `Metrics` -- Key should be reference to MetricNames entry +Add new object to `METRICS` +- Key should be reference to METRIC_NAMES entry - Value should be an object containing the metricType and metricConfig - for metricConfig, specify required config for metricType. @@ -127,18 +127,18 @@ See below API to record new sample for metric. Definition inside `prometheus.constants.js` ``` -let MetricNames = { +let METRIC_NAMES = { ... SYNC_QUEUE_JOBS_TOTAL_GAUGE: 'sync_queue_jobs_total', ... } -const Metrics = Object.freeze({ +const METRICS = Object.freeze({ ... - [MetricNames.SYNC_QUEUE_JOBS_TOTAL_GAUGE]: { - metricType: MetricTypes.GAUGE, + [METRIC_NAMES.SYNC_QUEUE_JOBS_TOTAL_GAUGE]: { + metricType: METRIC_TYPES.GAUGE, metricConfig: { - name: MetricNames.SYNC_QUEUE_JOBS_TOTAL_GAUGE, + name: METRIC_NAMES.SYNC_QUEUE_JOBS_TOTAL_GAUGE, help: 'Current job counts for SyncQueue by status', labelNames: ['status'] } diff --git a/creator-node/src/services/prometheusMonitoring/prometheus.constants.js b/creator-node/src/services/prometheusMonitoring/prometheus.constants.js index 208dc49273a..5550856bcc4 100644 --- a/creator-node/src/services/prometheusMonitoring/prometheus.constants.js +++ b/creator-node/src/services/prometheusMonitoring/prometheus.constants.js @@ -9,16 +9,16 @@ const { } = require('../stateMachineManager/stateMachineConstants') /** - * For explanation of Metrics, and instructions on how to add a new metric, please see `prometheusMonitoring/README.md` + * For explanation of METRICS, and instructions on how to add a new metric, please see `prometheusMonitoring/README.md` */ // We add a namespace prefix to differentiate internal metrics from those exported by different exporters from the same host -const NamespacePrefix = 'audius_cn_' +const NAMESPACE_PREFIX = 'audius_cn' /** * @notice Counter and Summary metric types are currently disabled, see README for details. */ -const MetricTypes = Object.freeze({ +const METRIC_TYPES = Object.freeze({ GAUGE: promClient.Gauge, HISTOGRAM: promClient.Histogram // COUNTER: promClient.Counter, @@ -28,15 +28,13 @@ const MetricTypes = Object.freeze({ /** * Types for recording a metric value. */ -const MetricRecordType = Object.freeze({ +const METRIC_RECORD_TYPE = Object.freeze({ GAUGE_INC: 'GAUGE_INC', HISTOGRAM_OBSERVE: 'HISTOGRAM_OBSERVE' }) -let MetricNames = { +const metricNames = { SYNC_QUEUE_JOBS_TOTAL_GAUGE: 'sync_queue_jobs_total', - ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM: - 'route_post_tracks_duration_seconds', ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM: 'issue_sync_request_duration_seconds', FIND_SYNC_REQUEST_COUNTS_GAUGE: 'find_sync_request_counts', @@ -45,16 +43,16 @@ let MetricNames = { // Add a histogram for each job in the state machine queues. // Some have custom labels below, and all of them use the label: uncaughtError=true/false for (const jobName of Object.values(STATE_MACHINE_JOB_NAMES)) { - MetricNames[ + metricNames[ `STATE_MACHINE_${jobName}_JOB_DURATION_SECONDS_HISTOGRAM` ] = `state_machine_${_.snakeCase(jobName)}_job_duration_seconds` } -MetricNames = Object.freeze( - _.mapValues(MetricNames, (metricName) => NamespacePrefix + metricName) +const METRIC_NAMES = Object.freeze( + _.mapValues(metricNames, (metricName) => `${NAMESPACE_PREFIX}_${metricName}`) ) -const MetricLabels = Object.freeze({ - [MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM]: { +const METRIC_LABELS = Object.freeze({ + [METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM]: { sync_type: Object.values(SyncType).map(_.snakeCase), sync_mode: Object.values(SYNC_MODES).map(_.snakeCase), result: [ @@ -70,7 +68,7 @@ const MetricLabels = Object.freeze({ ] }, - [MetricNames[ + [METRIC_NAMES[ `STATE_MACHINE_${STATE_MACHINE_JOB_NAMES.UPDATE_REPLICA_SET}_JOB_DURATION_SECONDS_HISTOGRAM` ]]: { // Whether or not the user's replica set was updated during this job @@ -84,7 +82,7 @@ const MetricLabels = Object.freeze({ ] }, - [MetricNames.FIND_SYNC_REQUEST_COUNTS_GAUGE]: { + [METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE]: { sync_mode: Object.values(SYNC_MODES).map(_.snakeCase), result: [ 'not_checked', // Default value -- means the logic short-circuited before checking if the primary should sync to the secondary. This can be expected if this node wasn't the user's primary @@ -100,7 +98,7 @@ const MetricLabels = Object.freeze({ ] }, - [MetricNames.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM]: { + [METRIC_NAMES.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM]: { // Whether or not write quorum is enabled/enforced enforceWriteQuorum: ['false', 'true'], // Whether or not write quorum is ignored for this specific route (even if it's enabled in general). @@ -129,42 +127,31 @@ const MetricLabels = Object.freeze({ const MetricLabelNames = Object.freeze( Object.fromEntries( - Object.entries(MetricLabels).map(([metric, metricLabels]) => [ + Object.entries(METRIC_LABELS).map(([metric, metricLabels]) => [ metric, Object.keys(metricLabels) ]) ) ) -const Metrics = Object.freeze({ - [MetricNames.SYNC_QUEUE_JOBS_TOTAL_GAUGE]: { - metricType: MetricTypes.GAUGE, +const METRICS = Object.freeze({ + [METRIC_NAMES.SYNC_QUEUE_JOBS_TOTAL_GAUGE]: { + metricType: METRIC_TYPES.GAUGE, metricConfig: { - name: MetricNames.SYNC_QUEUE_JOBS_TOTAL_GAUGE, + name: METRIC_NAMES.SYNC_QUEUE_JOBS_TOTAL_GAUGE, help: 'Current job counts for SyncQueue by status', labelNames: ['status'] } }, - /** @notice This metric will eventually be replaced by an express route metrics middleware */ - [MetricNames.ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM]: { - metricType: MetricTypes.HISTOGRAM, + [METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM]: { + metricType: METRIC_TYPES.HISTOGRAM, metricConfig: { - name: MetricNames.ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM, - help: 'Duration for POST /tracks route', - labelNames: ['code'], - buckets: [0.1, 0.3, 0.5, 1, 3, 5, 10] // 0.1 to 10 seconds - } - }, - - [MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM]: { - metricType: MetricTypes.HISTOGRAM, - metricConfig: { - name: MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM, + name: METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM, help: 'Time spent to issue a sync request and wait for completion (seconds)', labelNames: MetricLabelNames[ - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ], // 4 buckets in the range of 1 second to max before timing out a sync request buckets: exponentialBucketsRange( @@ -178,11 +165,11 @@ const Metrics = Object.freeze({ // Add histogram for each job in the state machine queues ...Object.fromEntries( Object.values(STATE_MACHINE_JOB_NAMES).map((jobName) => [ - MetricNames[`STATE_MACHINE_${jobName}_JOB_DURATION_SECONDS_HISTOGRAM`], + METRIC_NAMES[`STATE_MACHINE_${jobName}_JOB_DURATION_SECONDS_HISTOGRAM`], { - metricType: MetricTypes.HISTOGRAM, + metricType: METRIC_TYPES.HISTOGRAM, metricConfig: { - name: MetricNames[ + name: METRIC_NAMES[ `STATE_MACHINE_${jobName}_JOB_DURATION_SECONDS_HISTOGRAM` ], help: `Duration in seconds for a ${jobName} job to complete`, @@ -191,7 +178,7 @@ const Metrics = Object.freeze({ 'uncaughtError', // Label names, if any, that are specific to this job type ...(MetricLabelNames[ - MetricNames[ + METRIC_NAMES[ `STATE_MACHINE_${jobName}_JOB_DURATION_SECONDS_HISTOGRAM` ] ] || []) @@ -201,23 +188,21 @@ const Metrics = Object.freeze({ } ]) ), - - [MetricNames.FIND_SYNC_REQUEST_COUNTS_GAUGE]: { - metricType: MetricTypes.GAUGE, + [METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE]: { + metricType: METRIC_TYPES.GAUGE, metricConfig: { - name: MetricNames.FIND_SYNC_REQUEST_COUNTS_GAUGE, + name: METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE, help: "Counts for each find-sync-requests job's result when looking for syncs that should be requested from a primary to a secondary", - labelNames: MetricLabelNames[MetricNames.FIND_SYNC_REQUEST_COUNTS_GAUGE] + labelNames: MetricLabelNames[METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE] } }, - - [MetricNames.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM]: { - metricType: MetricTypes.HISTOGRAM, + [METRIC_NAMES.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM]: { + metricType: METRIC_TYPES.HISTOGRAM, metricConfig: { - name: MetricNames.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM, + name: METRIC_NAMES.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM, help: 'Seconds spent attempting to replicate data to a secondary node for write quorum', labelNames: - MetricLabelNames[MetricNames.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM], + MetricLabelNames[METRIC_NAMES.WRITE_QUORUM_DURATION_SECONDS_HISTOGRAM], // 5 buckets in the range of 1 second to max seconds before timing out write quorum buckets: exponentialBucketsRange( 1, @@ -229,9 +214,11 @@ const Metrics = Object.freeze({ } }) -module.exports.NamespacePrefix = NamespacePrefix -module.exports.MetricTypes = MetricTypes -module.exports.MetricNames = MetricNames -module.exports.MetricLabels = MetricLabels -module.exports.MetricRecordType = MetricRecordType -module.exports.Metrics = Metrics +module.exports = { + NAMESPACE_PREFIX, + METRIC_TYPES, + METRIC_NAMES, + METRIC_LABELS, + METRIC_RECORD_TYPE, + METRICS +} diff --git a/creator-node/src/services/prometheusMonitoring/prometheusRegistry.js b/creator-node/src/services/prometheusMonitoring/prometheusRegistry.js index c6901ca2335..f21b2aab322 100644 --- a/creator-node/src/services/prometheusMonitoring/prometheusRegistry.js +++ b/creator-node/src/services/prometheusMonitoring/prometheusRegistry.js @@ -1,32 +1,57 @@ const PrometheusClient = require('prom-client') const { - NamespacePrefix, - Metrics, - MetricNames + NAMESPACE_PREFIX, + METRICS, + METRIC_NAMES } = require('./prometheus.constants') /** * See `prometheusMonitoring/README.md` for usage details */ -module.exports = class PrometheusRegistry { +class PrometheusRegistry { constructor() { // Use default global registry to register metrics this.registry = PrometheusClient.register - // Expose metric names from class for access throughout application - this.metricNames = MetricNames - // Ensure clean state for registry this.registry.clear() // Enable collection of default metrics (e.g. heap, cpu, event loop) PrometheusClient.collectDefaultMetrics({ - prefix: NamespacePrefix + 'default_' + prefix: NAMESPACE_PREFIX + '_default_' }) - createAllCustomMetrics(this.registry) + this.initStaticMetrics(this.registry) + + // Expose metric names from class for access throughout application + this.metricNames = { ...METRIC_NAMES } + + this.namespacePrefix = NAMESPACE_PREFIX + + this.regexes = [] + } + + /** + * Creates and registers every static metric defined in prometheus.constants.js + */ + initStaticMetrics(registry) { + for (const { metricType: MetricType, metricConfig } of Object.values( + METRICS + )) { + // Create and register instance of MetricType, with provided metricConfig + const metric = new MetricType(metricConfig) + registry.registerMetric(metric) + } + } + + /** + * Initializes the regexes to match on routes with route params + * @param {RegExp[]} regexes the regexes used to match on routes with route params + */ + initRouteParamRegexes(regexes) { + this.regexes = regexes } /** Getters */ @@ -42,15 +67,4 @@ module.exports = class PrometheusRegistry { } } -/** - * Creates and registers every custom metric, for use throughout application - */ -const createAllCustomMetrics = function (registry) { - for (const { metricType: MetricType, metricConfig } of Object.values( - Metrics - )) { - // Create and register instance of MetricType, with provided metricConfig - const metric = new MetricType(metricConfig) - registry.registerMetric(metric) - } -} +module.exports = PrometheusRegistry diff --git a/creator-node/src/services/stateMachineManager/makeOnCompleteCallback.js b/creator-node/src/services/stateMachineManager/makeOnCompleteCallback.js index 987d117f6b1..78a7f7489d0 100644 --- a/creator-node/src/services/stateMachineManager/makeOnCompleteCallback.js +++ b/creator-node/src/services/stateMachineManager/makeOnCompleteCallback.js @@ -1,7 +1,7 @@ const { logger: baseLogger, createChildLogger } = require('../../logging') const { QUEUE_NAMES, JOB_NAMES } = require('./stateMachineConstants') const { - MetricRecordType + METRIC_RECORD_TYPE } = require('../prometheusMonitoring/prometheus.constants') /** @@ -173,9 +173,9 @@ const recordMetrics = (prometheusRegistry, logger, metricsToRecord = []) => { try { const { metricName, metricType, metricValue, metricLabels } = metricInfo const metric = prometheusRegistry.getMetric(metricName) - if (metricType === MetricRecordType.HISTOGRAM_OBSERVE) { + if (metricType === METRIC_RECORD_TYPE.HISTOGRAM_OBSERVE) { metric.observe(metricLabels, metricValue) - } else if (metricType === MetricRecordType.GAUGE_INC) { + } else if (metricType === METRIC_RECORD_TYPE.GAUGE_INC) { metric.inc(metricLabels, metricValue) } else { logger.error(`Unexpected metric type: ${metricType}`) diff --git a/creator-node/src/services/stateMachineManager/stateMachineUtils.js b/creator-node/src/services/stateMachineManager/stateMachineUtils.js index 15cf950eb0f..ffe5f6cb08f 100644 --- a/creator-node/src/services/stateMachineManager/stateMachineUtils.js +++ b/creator-node/src/services/stateMachineManager/stateMachineUtils.js @@ -4,9 +4,9 @@ const axios = require('axios') const retry = require('async-retry') const { - MetricRecordType, - MetricNames, - MetricLabels + METRIC_RECORD_TYPE, + METRIC_NAMES, + METRIC_LABELS } = require('../../services/prometheusMonitoring/prometheus.constants') const config = require('../../config') const { logger } = require('../../logging') @@ -149,7 +149,7 @@ const retrieveClockValueForUserFromReplica = async (replica, wallet) => { */ const makeHistogramToRecord = (metricName, metricValue, metricLabels = {}) => { return makeMetricToRecord( - MetricRecordType.HISTOGRAM_OBSERVE, + METRIC_RECORD_TYPE.HISTOGRAM_OBSERVE, metricName, metricValue, metricLabels @@ -166,7 +166,7 @@ const makeHistogramToRecord = (metricName, metricValue, metricLabels = {}) => { */ const makeGaugeIncToRecord = (metricName, incBy, metricLabels = {}) => { return makeMetricToRecord( - MetricRecordType.GAUGE_INC, + METRIC_RECORD_TYPE.GAUGE_INC, metricName, incBy, metricLabels @@ -176,7 +176,7 @@ const makeGaugeIncToRecord = (metricName, incBy, metricLabels = {}) => { /** * Returns an object that can be returned from any state machine job to record a change in a metric. * Validates the params to make sure the metric is valid. - * @param {string} metricType the type of metric being recorded -- HISTOGRAM_OBSERVE or GAUGE_INC + * @param {string} metricType the type of metric being recorded -- HISTOGRAM or GAUGE_INC * @param {string} metricName the name of the metric from prometheus.constants * @param {number} metricValue the value to observe * @param {string} [metricLabels] the optional mapping of metric label name => metric label value @@ -187,21 +187,21 @@ const makeMetricToRecord = ( metricValue, metricLabels = {} ) => { - if (!Object.values(MetricRecordType).includes(metricType)) { + if (!Object.values(METRIC_RECORD_TYPE).includes(metricType)) { throw new Error(`Invalid metricType: ${metricType}`) } - if (!Object.values(MetricNames).includes(metricName)) { + if (!Object.values(METRIC_NAMES).includes(metricName)) { throw new Error(`Invalid metricName: ${metricName}`) } if (typeof metricValue !== 'number') { throw new Error(`Invalid non-numerical metricValue: ${metricValue}`) } - const labelNames = Object.keys(MetricLabels[metricName]) + const labelNames = Object.keys(METRIC_LABELS[metricName]) for (const [labelName, labelValue] of Object.entries(metricLabels)) { if (!labelNames?.includes(labelName)) { throw new Error(`Metric label has invalid name: ${labelName}`) } - const labelValues = MetricLabels[metricName][labelName] + const labelValues = METRIC_LABELS[metricName][labelName] if (!labelValues?.includes(labelValue) && labelValues?.length !== 0) { throw new Error(`Metric label has invalid value: ${labelValue}`) } diff --git a/creator-node/src/services/stateMachineManager/stateMonitoring/findSyncRequests.jobProcessor.js b/creator-node/src/services/stateMachineManager/stateMonitoring/findSyncRequests.jobProcessor.js index 4a411a090fd..d7efb728707 100644 --- a/creator-node/src/services/stateMachineManager/stateMonitoring/findSyncRequests.jobProcessor.js +++ b/creator-node/src/services/stateMachineManager/stateMonitoring/findSyncRequests.jobProcessor.js @@ -2,7 +2,7 @@ const _ = require('lodash') const config = require('../../../config') const { - MetricNames + METRIC_NAMES } = require('../../prometheusMonitoring/prometheus.constants') const CNodeToSpIdMapManager = require('../CNodeToSpIdMapManager') const { makeGaugeIncToRecord } = require('../stateMachineUtils') @@ -119,7 +119,7 @@ module.exports = async function ({ for (const [result, count] of Object.entries(resultCountsMap)) { metricsToRecord.push( makeGaugeIncToRecord( - MetricNames.FIND_SYNC_REQUEST_COUNTS_GAUGE, + METRIC_NAMES.FIND_SYNC_REQUEST_COUNTS_GAUGE, count, { sync_mode: _.snakeCase(syncMode), result } ) diff --git a/creator-node/src/services/stateMachineManager/stateReconciliation/issueSyncRequest.jobProcessor.js b/creator-node/src/services/stateMachineManager/stateReconciliation/issueSyncRequest.jobProcessor.js index 0f5c11ee2a9..ea814f584e1 100644 --- a/creator-node/src/services/stateMachineManager/stateReconciliation/issueSyncRequest.jobProcessor.js +++ b/creator-node/src/services/stateMachineManager/stateReconciliation/issueSyncRequest.jobProcessor.js @@ -5,7 +5,7 @@ const config = require('../../../config') const models = require('../../../models') const Utils = require('../../../utils') const { - MetricNames + METRIC_NAMES } = require('../../prometheusMonitoring/prometheus.constants') const { retrieveClockValueForUserFromReplica, @@ -105,7 +105,7 @@ module.exports = async function ({ // Make metrics to record metricsToRecord = [ makeHistogramToRecord( - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM, + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM, (Date.now() - startTimeMs) / 1000, // Metric is in seconds { sync_type: _.snakeCase(syncType), diff --git a/creator-node/test/issueSyncRequest.jobProcessor.test.js b/creator-node/test/issueSyncRequest.jobProcessor.test.js index 68f08e32ba1..8d7e0f47692 100644 --- a/creator-node/test/issueSyncRequest.jobProcessor.test.js +++ b/creator-node/test/issueSyncRequest.jobProcessor.test.js @@ -11,13 +11,11 @@ const { getLibsMock } = require('./lib/libsMock') const models = require('../src/models') const config = require('../src/config') const stateMachineConstants = require('../src/services/stateMachineManager/stateMachineConstants') -const { - SyncType, - QUEUE_NAMES, - SYNC_MODES -} = stateMachineConstants +const { SyncType, QUEUE_NAMES, SYNC_MODES } = stateMachineConstants const issueSyncRequestJobProcessor = require('../src/services/stateMachineManager/stateReconciliation/issueSyncRequest.jobProcessor') -const { MetricNames } = require('../src/services/prometheusMonitoring/prometheus.constants') +const { + METRIC_NAMES +} = require('../src/services/prometheusMonitoring/prometheus.constants') chai.use(require('sinon-chai')) chai.use(require('chai-as-promised')) @@ -26,7 +24,7 @@ const { expect } = chai describe('test issueSyncRequest job processor param validation', function () { let server, sandbox, originalContentNodeEndpoint, logger - let syncMode = SYNC_MODES.SyncSecondaryFromPrimary + const syncMode = SYNC_MODES.SyncSecondaryFromPrimary beforeEach(async function () { const appInfo = await getApp(getLibsMock()) @@ -66,7 +64,9 @@ describe('test issueSyncRequest job processor param validation', function () { const syncType = SyncType.Manual - const expectedErrorMessage = `Invalid sync data found: ${JSON.stringify(syncRequestParameters)}` + const expectedErrorMessage = `Invalid sync data found: ${JSON.stringify( + syncRequestParameters + )}` // Verify job outputs the correct results: sync to user1 to secondary1 because its clock value is behind const result = await issueSyncRequestJobProcessor({ @@ -81,7 +81,7 @@ describe('test issueSyncRequest job processor param validation', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_mode: _.snakeCase(syncMode), @@ -128,7 +128,7 @@ describe('test issueSyncRequest job processor param validation', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_mode: _.snakeCase(syncMode), @@ -168,7 +168,7 @@ describe('test issueSyncRequest job processor', function () { wallet = '0x123456789' secondary = 'http://some_cn.co' - data = { wallet: [wallet] } + data = { wallet: [wallet] } syncRequestParameters = { baseURL: secondary, url: '/sync', @@ -208,7 +208,6 @@ describe('test issueSyncRequest job processor', function () { retrieveClockValueForUserFromReplicaStub, primarySyncFromSecondaryStub }) { - const stubs = { '../../../config': config, './stateReconciliationUtils': { @@ -226,12 +225,13 @@ describe('test issueSyncRequest job processor', function () { }, '../stateMachineConstants': { ...stateMachineConstants, - SYNC_MONITORING_RETRY_DELAY_MS: 1, + SYNC_MONITORING_RETRY_DELAY_MS: 1 } } if (primarySyncFromSecondaryStub) { - stubs['../../sync/primarySyncFromSecondary'] = primarySyncFromSecondaryStub + stubs['../../sync/primarySyncFromSecondary'] = + primarySyncFromSecondaryStub } return proxyquire( @@ -275,7 +275,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_type: _.snakeCase(syncType), @@ -327,7 +327,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_type: _.snakeCase(syncType), @@ -354,7 +354,12 @@ describe('test issueSyncRequest job processor', function () { const expectedSyncReqToEnqueue = 'expectedSyncReqToEnqueue' const getNewOrExistingSyncReqStub = sandbox.stub().callsFake((args) => { - const { userWallet, secondaryEndpoint, syncType: syncTypeArg, syncMode: syncModeArg } = args + const { + userWallet, + secondaryEndpoint, + syncType: syncTypeArg, + syncMode: syncModeArg + } = args if ( userWallet === wallet && secondaryEndpoint === secondary && @@ -406,7 +411,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_type: _.snakeCase(syncType), @@ -444,7 +449,12 @@ describe('test issueSyncRequest job processor', function () { const expectedSyncReqToEnqueue = 'expectedSyncReqToEnqueue' const getNewOrExistingSyncReqStub = sandbox.stub().callsFake((args) => { - const { userWallet, secondaryEndpoint, syncType: syncTypeArg, syncMode: syncModeArg } = args + const { + userWallet, + secondaryEndpoint, + syncType: syncTypeArg, + syncMode: syncModeArg + } = args if ( userWallet === wallet && secondaryEndpoint === secondary && @@ -493,7 +503,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_mode: _.snakeCase(syncMode), @@ -560,7 +570,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_mode: _.snakeCase(syncMode), @@ -584,12 +594,14 @@ describe('test issueSyncRequest job processor', function () { const getNewOrExistingSyncReqStub = sandbox.stub().callsFake((args) => { throw new Error('getNewOrExistingSyncReq was not expected to be called') }) - + const getSecondaryUserSyncFailureCountForTodayStub = sandbox .stub() .returns(0) - - const retrieveClockValueForUserFromReplicaStub = sandbox.stub().resolves(1) + + const retrieveClockValueForUserFromReplicaStub = sandbox + .stub() + .resolves(1) config.set('mergePrimaryAndSecondaryEnabled', true) @@ -598,20 +610,22 @@ describe('test issueSyncRequest job processor', function () { if (walletParam === wallet && secondaryParam === secondary) { return } - throw new Error(`primarySyncFromSecondary was not expected to be called with the given args`) + throw new Error( + `primarySyncFromSecondary was not expected to be called with the given args` + ) }) - + const issueSyncRequestJobProcessor = getJobProcessorStub({ getNewOrExistingSyncReqStub, getSecondaryUserSyncFailureCountForTodayStub, retrieveClockValueForUserFromReplicaStub, primarySyncFromSecondaryStub }) - + // Make the axios request succeed const syncReqData = { ...data, forceResync: true } nock(secondary).post('/sync', syncReqData).reply(200) - + // Verify job outputs the correct results: no sync issued (nock will error if the wrong network req was made) const result = await issueSyncRequestJobProcessor({ logger, @@ -624,7 +638,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_type: _.snakeCase(syncType), @@ -647,12 +661,14 @@ describe('test issueSyncRequest job processor', function () { const getNewOrExistingSyncReqStub = sandbox.stub().callsFake((args) => { throw new Error('getNewOrExistingSyncReq was not expected to be called') }) - + const getSecondaryUserSyncFailureCountForTodayStub = sandbox .stub() .returns(0) - - const retrieveClockValueForUserFromReplicaStub = sandbox.stub().resolves(1) + + const retrieveClockValueForUserFromReplicaStub = sandbox + .stub() + .resolves(1) config.set('mergePrimaryAndSecondaryEnabled', true) @@ -662,16 +678,18 @@ describe('test issueSyncRequest job processor', function () { if (walletParam === wallet && secondaryParam === secondary) { return primarySyncFromSecondaryError } - throw new Error(`primarySyncFromSecondary was not expected to be called with the given args`) + throw new Error( + `primarySyncFromSecondary was not expected to be called with the given args` + ) }) - + const issueSyncRequestJobProcessor = getJobProcessorStub({ getNewOrExistingSyncReqStub, getSecondaryUserSyncFailureCountForTodayStub, retrieveClockValueForUserFromReplicaStub, - primarySyncFromSecondaryStub, + primarySyncFromSecondaryStub }) - + // Verify job outputs the correct results: no sync issued const result = await issueSyncRequestJobProcessor({ logger, @@ -679,12 +697,15 @@ describe('test issueSyncRequest job processor', function () { syncMode, syncRequestParameters }) - expect(result).to.have.deep.property('error', primarySyncFromSecondaryError) + expect(result).to.have.deep.property( + 'error', + primarySyncFromSecondaryError + ) expect(result).to.have.deep.property('jobsToEnqueue', {}) expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_mode: _.snakeCase(syncMode), @@ -703,29 +724,33 @@ describe('test issueSyncRequest job processor', function () { const getNewOrExistingSyncReqStub = sandbox.stub().callsFake((args) => { throw new Error('getNewOrExistingSyncReq was not expected to be called') }) - + const getSecondaryUserSyncFailureCountForTodayStub = sandbox .stub() .returns(0) - - const retrieveClockValueForUserFromReplicaStub = sandbox.stub().resolves(1) + + const retrieveClockValueForUserFromReplicaStub = sandbox + .stub() + .resolves(1) config.set('mergePrimaryAndSecondaryEnabled', false) const primarySyncFromSecondaryStub = sandbox.stub().callsFake((args) => { - throw new Error(`primarySyncFromSecondary was not expected to be called with the given args`) + throw new Error( + `primarySyncFromSecondary was not expected to be called with the given args` + ) }) - + const issueSyncRequestJobProcessor = getJobProcessorStub({ getNewOrExistingSyncReqStub, getSecondaryUserSyncFailureCountForTodayStub, retrieveClockValueForUserFromReplicaStub, primarySyncFromSecondaryStub }) - + // Make the axios request succeed nock(secondary).post('/sync', data).reply(200) - + // Verify job outputs the correct results: no sync issued (nock will error if the wrong network req was made) const result = await issueSyncRequestJobProcessor({ logger, @@ -738,7 +763,7 @@ describe('test issueSyncRequest job processor', function () { expect(result.metricsToRecord).to.have.lengthOf(1) expect(result.metricsToRecord[0]).to.have.deep.property( 'metricName', - MetricNames.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM + METRIC_NAMES.ISSUE_SYNC_REQUEST_DURATION_SECONDS_HISTOGRAM ) expect(result.metricsToRecord[0]).to.have.deep.property('metricLabels', { sync_mode: _.snakeCase(syncMode), @@ -753,6 +778,8 @@ describe('test issueSyncRequest job processor', function () { expect(getNewOrExistingSyncReqStub).to.not.have.been.called }) - it.skip('requires additional sync when secondary updates clock value but is still behind primary') + it.skip( + 'requires additional sync when secondary updates clock value but is still behind primary' + ) }) }) diff --git a/creator-node/test/lib/app.js b/creator-node/test/lib/app.js index 0af24d38c69..fb68243c4e6 100644 --- a/creator-node/test/lib/app.js +++ b/creator-node/test/lib/app.js @@ -8,7 +8,12 @@ const TrustedNotifierManager = require('../../src/services/TrustedNotifierManage const PrometheusRegistry = require('../../src/services/prometheusMonitoring/prometheusRegistry') const BlacklistManager = require('../../src/blacklistManager') -async function getApp (libsClient, blacklistManager = new BlacklistManager(), setMockFn = null, spId = null) { +async function getApp( + libsClient, + blacklistManager = BlacklistManager, + setMockFn = null, + spId = null +) { // we need to clear the cache that commonjs require builds, otherwise it uses old values for imports etc // eg if you set a new env var, it doesn't propogate well unless you clear the cache for the config file as well // as all files that consume it @@ -22,7 +27,7 @@ async function getApp (libsClient, blacklistManager = new BlacklistManager(), se const mockServiceRegistry = { libs: libsClient, - blacklistManager: blacklistManager, + blacklistManager, redis: redisClient, monitoringQueue: new MonitoringQueueMock(), asyncProcessingQueue: new AsyncProcessingQueueMock(), @@ -45,7 +50,7 @@ async function getApp (libsClient, blacklistManager = new BlacklistManager(), se return appInfo } -function getServiceRegistryMock (libsClient, blacklistManager) { +function getServiceRegistryMock(libsClient, blacklistManager) { return { libs: libsClient, blacklistManager: blacklistManager, @@ -57,12 +62,15 @@ function getServiceRegistryMock (libsClient, blacklistManager) { } } -function clearRequireCache () { +function clearRequireCache() { console.log('DELETING CACHE') Object.keys(require.cache).forEach(function (key) { // exclude src/models/index from the key deletion because it initalizes a new connection pool // every time and we hit a db error if we clear the cache and keep creating new pg pools - if (key.includes('creator-node/src/') && !key.includes('creator-node/src/models/index.js')) { + if ( + key.includes('creator-node/src/') && + !key.includes('creator-node/src/models/index.js') + ) { delete require.cache[key] } }) diff --git a/creator-node/test/prometheus.test.js b/creator-node/test/prometheus.test.js index b57ebbdc3c2..c2afc39e0cb 100644 --- a/creator-node/test/prometheus.test.js +++ b/creator-node/test/prometheus.test.js @@ -3,8 +3,9 @@ const request = require('supertest') const { getApp } = require('./lib/app') const { getLibsMock } = require('./lib/libsMock') - -const { NamespacePrefix } = require('../src/services/prometheusMonitoring/prometheus.constants') +const { + NAMESPACE_PREFIX +} = require('../src/services/prometheusMonitoring/prometheus.constants') describe('test Prometheus metrics', async function () { let app, server, libsMock @@ -14,6 +15,7 @@ describe('test Prometheus metrics', async function () { libsMock = getLibsMock() const appInfo = await getApp(libsMock) + app = appInfo.app server = appInfo.server }) @@ -22,10 +24,40 @@ describe('test Prometheus metrics', async function () { await server.close() }) - it('Checks that GET /prometheus_metrics is healthy and exposes Default metrics', async function () { - const resp = await request(app) - .get('/prometheus_metrics') - .expect(200) - assert.ok(resp.text.includes(NamespacePrefix + 'default_' + 'process_cpu_user_seconds_total')) + it('Checks that GET /prometheus_metrics is healthy and exposes default metrics', async function () { + await request(app).get('/health_check') + + const resp = await request(app).get('/prometheus_metrics').expect(200) + assert.ok( + resp.text.includes( + NAMESPACE_PREFIX + '_default_' + 'process_cpu_user_seconds_total' + ) + ) + + assert.ok( + resp.text.includes(NAMESPACE_PREFIX + '_http_request_duration_seconds') + ) + }) + + it('Checks the middleware tracks routes with route params', async function () { + app.get('serviceRegistry').prometheusRegistry.regexes = [ + { + regex: /(?:^\/ipfs\/(?:([^/]+?))\/?$|^\/content\/(?:([^/]+?))\/?$)/i, + path: '/ipfs/#CID' + } + ] + + await request(app).get('/ipfs/QmVickyWasHere') + await request(app).get('/content/QmVickyWasHere') + + const resp = await request(app).get('/prometheus_metrics').expect(200) + + assert.ok( + resp.text.includes(NAMESPACE_PREFIX + '_http_request_duration_seconds') + ) + + assert.ok(resp.text.includes('/ipfs/#CID')) + + assert.ok(!resp.text.includes('/content/#CID')) }) })