fix(prometheus): metrics.ts leaks to global registry #1202

1. Specified a `register` property of the gauges as an empty
array so that it does not pollute the global namespace. This
is how this is supposed to be done as per the docs of prom-client.

2. Once the change from 1) took place, the issue became that
the metrics gathering code was still trying to hit up the
global scope for the metrics, e.g. calling the get metrics
methods directly on the promClient object instead of the
registry that we create for each prmoetheus exporter object
separately. So a little additional refactor ensued to fix this
as well by making sure that we grab a reference of the registry
object at construction time and then re-use that wherever needed
instead of going through the global promClient object.

3. Added missing .startMetricsCollection calls in the plugin
constructors to ensure that the prometheus exporter object
gets initialized properly (this is where the registry gets
created as well so without this there are crashes happening
when one tries to access the metrics through the registry)

Why though?
The problem was that the metrics.ts file that we have for all the
plugin's metrics constructs a new Metric (Gauge) object at import
time which then defaults to registering the metric in the global
registry of prom-client by default.

The latter was causing crashes when separate versions of the same
metrics.ts file are imported in a scenario were the API server
imports plugins from a different directory (this issue is coming
from the branch where I'm working on plugin sandboxing via the
live-plugin-manager).

Fixes #1202

Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>

packages/cactus-cmd-api-server/src/main/typescript/api-server.ts

@@ -120,6 +120,7 @@ export class ApiServer {
         pollingIntervalInMin: 1,
       });
     }
+    this.prometheusExporter.startMetricsCollection();
     this.prometheusExporter.setTotalPluginImports(this.getPluginImportsCount());
 
     this.log = LoggerProvider.getOrCreate({

packages/cactus-cmd-api-server/src/main/typescript/prometheus-exporter/metrics.ts

@@ -4,6 +4,7 @@ export const K_CACTUS_API_SERVER_TOTAL_PLUGIN_IMPORTS =
   "cactus_api_server_total_plugin_imports";
 
 export const totalTxCount = new Gauge({
+  registers: [],
   name: K_CACTUS_API_SERVER_TOTAL_PLUGIN_IMPORTS,
   help: "Total number of plugins imported",
   labelNames: ["type"],

packages/cactus-cmd-api-server/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,6 +1,7 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { TotalPluginImports } from "./response.type";
 import { K_CACTUS_API_SERVER_TOTAL_PLUGIN_IMPORTS } from "./metrics";
+import { totalTxCount } from "./metrics";
 import { collectMetrics } from "./data-fetcher";
 
 export interface IPrometheusExporterOptions {
@@ -10,12 +11,14 @@ export interface IPrometheusExporterOptions {
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly totalPluginImports: TotalPluginImports = { counter: 0 };
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public setTotalPluginImports(totalPluginImports: number): void {
@@ -24,15 +27,14 @@ export class PrometheusExporter {
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_API_SERVER_TOTAL_PLUGIN_IMPORTS,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalTxCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-consortium-manual/src/main/typescript/plugin-consortium-manual.ts

@@ -84,6 +84,7 @@ export class PluginConsortiumManual
       this.prometheusExporter,
       `${fnTag} options.prometheusExporter`,
     );
+    this.prometheusExporter.startMetricsCollection();
     this.prometheusExporter.setNodeCount(this.getNodeCount());
   }
 

packages/cactus-plugin-consortium-manual/src/main/typescript/prometheus-exporter/metrics.ts

@@ -4,6 +4,7 @@ export const K_CACTUS_CONSORTIUM_MANUAL_TOTAL_NODE_COUNT =
   "cactus_consortium_manual_total_node_count";
 
 export const totalTxCount = new Gauge({
+  registers: [],
   name: K_CACTUS_CONSORTIUM_MANUAL_TOTAL_NODE_COUNT,
   help: "Total cactus node count",
   labelNames: ["type"],

packages/cactus-plugin-consortium-manual/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,4 +1,4 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { NodeCount } from "./response.type";
 import {
   totalTxCount,
@@ -12,12 +12,14 @@ export interface IPrometheusExporterOptions {
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly nodeCount: NodeCount = { counter: 0 };
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public setNodeCount(nodeCount: number): void {
@@ -28,15 +30,14 @@ export class PrometheusExporter {
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_CONSORTIUM_MANUAL_TOTAL_NODE_COUNT,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalTxCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-keychain-memory/src/main/typescript/plugin-keychain-memory.ts

@@ -58,6 +58,7 @@ export class PluginKeychainMemory {
       this.prometheusExporter,
       `${fnTag} options.prometheusExporter`,
     );
+    this.prometheusExporter.startMetricsCollection();
 
     this.log.info(`Created ${this.className}. KeychainID=${opts.keychainId}`);
     this.log.warn(

packages/cactus-plugin-keychain-memory/src/main/typescript/prometheus-exporter/metrics.ts

@@ -4,6 +4,7 @@ export const K_CACTUS_KEYCHAIN_MEMORY_TOTAL_KEY_COUNT =
   "cactus_keychain_memory_total_key_count";
 
 export const totalKeyCount = new Gauge({
+  registers: [],
   name: K_CACTUS_KEYCHAIN_MEMORY_TOTAL_KEY_COUNT,
   help: "Total keys present in memory",
   labelNames: ["type"],

packages/cactus-plugin-keychain-memory/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,20 +1,24 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { KeyCount } from "./response.type";
 import { collectMetrics } from "./data-fetcher";
 import { K_CACTUS_KEYCHAIN_MEMORY_TOTAL_KEY_COUNT } from "./metrics";
+import { totalKeyCount } from "./metrics";
+
 export interface IPrometheusExporterOptions {
   pollingIntervalInMin?: number;
 }
 
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly keyCount: KeyCount = { counter: 0 };
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public setTotalKeyCounter(keyCount: number): void {
@@ -23,15 +27,14 @@ export class PrometheusExporter {
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_KEYCHAIN_MEMORY_TOTAL_KEY_COUNT,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalKeyCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-keychain-vault/src/main/typescript/plugin-keychain-vault.ts

@@ -114,6 +114,7 @@ export class PluginKeychainVault implements IPluginWebService, IPluginKeychain {
       this.prometheusExporter,
       `${fnTag} options.prometheusExporter`,
     );
+    this.prometheusExporter.startMetricsCollection();
 
     this.log.info(`Created Vault backend OK. Endpoint=${this.endpoint}`);
 

packages/cactus-plugin-keychain-vault/src/main/typescript/prometheus-exporter/metrics.ts

@@ -4,6 +4,7 @@ export const K_CACTUS_KEYCHAIN_VAULT_MANAGED_KEY_COUNT =
   "cactus_keychain_vault_managed_key_count";
 
 export const totalKeyCount = new Gauge({
+  registers: [],
   name: K_CACTUS_KEYCHAIN_VAULT_MANAGED_KEY_COUNT,
   help:
     "The number of keys that were set in the backing Vault deployment via this specific keychain plugin instance",

packages/cactus-plugin-keychain-vault/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,7 +1,8 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { VaultKeys } from "./response.type";
 import { collectMetrics } from "./data-fetcher";
 import { K_CACTUS_KEYCHAIN_VAULT_MANAGED_KEY_COUNT } from "./metrics";
+import { totalKeyCount } from "./metrics";
 
 export interface IPrometheusExporterOptions {
   pollingIntervalInMin?: number;
@@ -10,12 +11,14 @@ export interface IPrometheusExporterOptions {
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly vaultKeys: VaultKeys = new Map();
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public setTotalKeyCounter(key: string, operation: string): void {
@@ -28,15 +31,14 @@ export class PrometheusExporter {
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_KEYCHAIN_VAULT_MANAGED_KEY_COUNT,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalKeyCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-ledger-connector-besu/src/main/typescript/prometheus-exporter/metrics.ts

@@ -3,6 +3,7 @@ import { Gauge } from "prom-client";
 export const K_CACTUS_BESU_TOTAL_TX_COUNT = "cactus_besu_total_tx_count";
 
 export const totalTxCount = new Gauge({
+  registers: [],
   name: "cactus_besu_total_tx_count",
   help: "Total transactions executed",
   labelNames: ["type"],

packages/cactus-plugin-ledger-connector-besu/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,7 +1,8 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { Transactions } from "./response.type";
 import { collectMetrics } from "./data-fetcher";
 import { K_CACTUS_BESU_TOTAL_TX_COUNT } from "./metrics";
+import { totalTxCount } from "./metrics";
 
 export interface IPrometheusExporterOptions {
   pollingIntervalInMin?: number;
@@ -10,28 +11,29 @@ export interface IPrometheusExporterOptions {
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly transactions: Transactions = { counter: 0 };
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public addCurrentTransaction(): void {
     collectMetrics(this.transactions);
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_BESU_TOTAL_TX_COUNT,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalTxCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-ledger-connector-corda/src/main/typescript/plugin-ledger-connector-corda.ts

@@ -77,6 +77,7 @@ export class PluginLedgerConnectorCorda
       this.prometheusExporter,
       `${fnTag} options.prometheusExporter`,
     );
+    this.prometheusExporter.startMetricsCollection();
   }
 
   public getPrometheusExporter(): PrometheusExporter {

packages/cactus-plugin-ledger-connector-corda/src/main/typescript/prometheus-exporter/metrics.ts

@@ -3,6 +3,7 @@ import { Gauge } from "prom-client";
 export const K_CACTUS_CORDA_TOTAL_TX_COUNT = "cactus_corda_total_tx_count";
 
 export const totalTxCount = new Gauge({
+  registers: [],
   name: K_CACTUS_CORDA_TOTAL_TX_COUNT,
   help: "Total transactions executed",
   labelNames: ["type"],

packages/cactus-plugin-ledger-connector-corda/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,4 +1,4 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { Transactions } from "./response.type";
 import { totalTxCount, K_CACTUS_CORDA_TOTAL_TX_COUNT } from "./metrics";
 
@@ -9,12 +9,14 @@ export interface IPrometheusExporterOptions {
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly transactions: Transactions = { counter: 0 };
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public addCurrentTransaction(): void {
@@ -25,15 +27,14 @@ export class PrometheusExporter {
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_CORDA_TOTAL_TX_COUNT,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalTxCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-ledger-connector-fabric/src/main/typescript/prometheus-exporter/prometheus-exporter.ts

@@ -1,4 +1,4 @@
-import promClient from "prom-client";
+import promClient, { Registry } from "prom-client";
 import { Transactions } from "./response.type";
 import { totalTxCount, K_CACTUS_FABRIC_TOTAL_TX_COUNT } from "./metrics";
 
@@ -9,12 +9,14 @@ export interface IPrometheusExporterOptions {
 export class PrometheusExporter {
   public readonly metricsPollingIntervalInMin: number;
   public readonly transactions: Transactions = { counter: 0 };
+  public readonly registry: Registry;
 
   constructor(
     public readonly prometheusExporterOptions: IPrometheusExporterOptions,
   ) {
     this.metricsPollingIntervalInMin =
       prometheusExporterOptions.pollingIntervalInMin || 1;
+    this.registry = new Registry();
   }
 
   public addCurrentTransaction(): void {
@@ -25,15 +27,14 @@ export class PrometheusExporter {
   }
 
   public async getPrometheusMetrics(): Promise<string> {
-    const result = await promClient.register.getSingleMetricAsString(
+    const result = await this.registry.getSingleMetricAsString(
       K_CACTUS_FABRIC_TOTAL_TX_COUNT,
     );
     return result;
   }
 
   public startMetricsCollection(): void {
-    const Registry = promClient.Registry;
-    const register = new Registry();
-    promClient.collectDefaultMetrics({ register });
+    this.registry.registerMetric(totalTxCount);
+    promClient.collectDefaultMetrics({ register: this.registry });
   }
 }

packages/cactus-plugin-ledger-connector-quorum/src/main/typescript/prometheus-exporter/metrics.ts

@@ -3,6 +3,7 @@ import { Gauge } from "prom-client";
 export const K_CACTUS_QUORUM_TOTAL_TX_COUNT = "cactus_quorum_total_tx_count";
 
 export const totalTxCount = new Gauge({
+  registers: [],
   name: K_CACTUS_QUORUM_TOTAL_TX_COUNT,
   help: "Total transactions executed",
   labelNames: ["type"],