From ab33a089e5e00a712d5240b9a4710acbb128f234 Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 6 Nov 2024 12:26:48 +0100 Subject: [PATCH 1/5] properly validate config `validate()` doesn't seem to work at all, replaced it with `joi.attempt()` that we use to validate the extension configs. Issue: BB-623 --- lib/Config.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/Config.js b/lib/Config.js index 4860a0dbe..dba169f4f 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -2,6 +2,7 @@ const assert = require('assert'); const { EventEmitter } = require('events'); +const joi = require('joi'); const fs = require('fs'); const path = require('path'); @@ -54,10 +55,7 @@ class Config extends EventEmitter { throw new Error(`could not parse config file: ${err.message}`); } - const { value: parsedConfig, err } = backbeatConfigJoi.validate(config); - if (err) { - throw new Error(`could not validate config file: ${err.message}`); - } + const parsedConfig = joi.attempt(config, backbeatConfigJoi); if (parsedConfig.extensions) { Object.keys(parsedConfig.extensions).forEach(extName => { From a8c1fb2e3cf920d0a71b6d5b6ebcb01153e28139 Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 6 Nov 2024 12:29:03 +0100 Subject: [PATCH 2/5] remove probeconfig validation from startProbeServer probeserver config validation should be done when validation the config, as it can be optional in some cases. Issue: BB-623 --- lib/util/probe.js | 3 +-- tests/unit/lib/util/probe.spec.js | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/util/probe.js b/lib/util/probe.js index 2274122a0..d214db424 100644 --- a/lib/util/probe.js +++ b/lib/util/probe.js @@ -26,8 +26,7 @@ const RdkafkaStats = require('node-rdkafka-prometheus'); */ function startProbeServer(config, callback) { if (!config) { - const err = new Error('configuration for probe server is missing'); - callback(err); + callback(); return; } diff --git a/tests/unit/lib/util/probe.spec.js b/tests/unit/lib/util/probe.spec.js index 133d816c5..fd893431d 100644 --- a/tests/unit/lib/util/probe.spec.js +++ b/tests/unit/lib/util/probe.spec.js @@ -4,10 +4,10 @@ const { startProbeServer, getReplicationProbeConfig } = const Logger = require('werelogs').Logger; describe('Probe server', () => { - it('is not created with no config', done => { + it('should return undefined when no config is passed', done => { const config = undefined; startProbeServer(config, (err, probeServer) => { - assert(err); + assert.ifError(err); assert.strictEqual(probeServer, undefined); done(); }); From d130b35b15188ada5fcd816a9d0ec3a072931425 Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 6 Nov 2024 12:31:22 +0100 Subject: [PATCH 3/5] make the probeserver config optional for the bucket notification processes In S3C, bucket notification processes only contain configuration for the notification extension. We make the probeserver config optional in this case as it is not currently supported in federation. Issue: BB-623 --- extensions/notification/NotificationConfigValidator.js | 4 +++- lib/config.joi.js | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/extensions/notification/NotificationConfigValidator.js b/extensions/notification/NotificationConfigValidator.js index fd7bc690e..924f89e94 100644 --- a/extensions/notification/NotificationConfigValidator.js +++ b/extensions/notification/NotificationConfigValidator.js @@ -33,7 +33,9 @@ const joiSchema = joi.object({ concurrency: joi.number().greater(0).default(1000), }, destinations: joi.array().items(destinationSchema).default([]), - probeServer: probeServerJoi.default() + // TODO: BB-625 reset to being required after supporting probeserver in S3C + // for bucket notification proceses + probeServer: probeServerJoi.optional() }); function configValidator(backbeatConfig, extConfig) { diff --git a/lib/config.joi.js b/lib/config.joi.js index a83d92a80..18a89862e 100644 --- a/lib/config.joi.js +++ b/lib/config.joi.js @@ -55,7 +55,13 @@ const joiSchema = joi.object({ }).when('logSource', { is: 'dmd', then: joi.required() }), mongo: mongoJoi, kafka: qpKafkaJoi.when('logSource', { is: 'kafka', then: joi.required() }), - probeServer: probeServerJoi.default(), + // TODO: BB-625 reset to being required after supporting probeserver in S3C + // for bucket notification proceses + probeServer: probeServerJoi.when('...extensions', { + is: joi.object().keys({ notification: joi.exist() }), + then: joi.optional(), + otherwise: joi.required(), + }), circuitBreaker: joi.object().optional(), }, log: logJoi, From 8465a7b7e9c947cb28a43c72abe6a9a6deaff92d Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 6 Nov 2024 12:34:16 +0100 Subject: [PATCH 4/5] test backbeat config validation Issue: BB-623 --- lib/Config.js | 9 + tests/unit/lib/config/Config.spec.js | 25 ++ tests/unit/lib/config/config.json | 364 +++++++++++++++++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 tests/unit/lib/config/Config.spec.js create mode 100644 tests/unit/lib/config/config.json diff --git a/lib/Config.js b/lib/Config.js index dba169f4f..290560576 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -55,6 +55,15 @@ class Config extends EventEmitter { throw new Error(`could not parse config file: ${err.message}`); } + this._parseConfig(config); + } + + /** + * Parses Backbeat's configuration + * @param {Object} config backbeat configuration + * @returns {undefined} + */ + _parseConfig(config) { const parsedConfig = joi.attempt(config, backbeatConfigJoi); if (parsedConfig.extensions) { diff --git a/tests/unit/lib/config/Config.spec.js b/tests/unit/lib/config/Config.spec.js new file mode 100644 index 000000000..f3160d20a --- /dev/null +++ b/tests/unit/lib/config/Config.spec.js @@ -0,0 +1,25 @@ +'use strict'; // eslint-disable-line + +const assert = require('assert'); + +const { Config } = require('../../../../lib/Config'); +const backbeatConfig = require('./config.json'); + +describe('Config', () => { + it('should make the probeserver config in the queuePoulator' + + 'required when multiple extensions are configured', () => { + const config = new Config(); + const testConfig = { ...backbeatConfig }; + delete testConfig.queuePopulator.probeServer; + assert.throws(() => config._parseConfig(testConfig)); + }); + + it('should make the probeserver config in the queuePoulator' + + 'optional when only notification config is specified', () => { + const config = new Config(); + const testConfig = { ...backbeatConfig }; + delete testConfig.queuePopulator.probeServer; + testConfig.extensions = { notification: testConfig.extensions.notification }; + assert.doesNotThrow(() => config._parseConfig(testConfig)); + }); +}); diff --git a/tests/unit/lib/config/config.json b/tests/unit/lib/config/config.json new file mode 100644 index 000000000..57b697014 --- /dev/null +++ b/tests/unit/lib/config/config.json @@ -0,0 +1,364 @@ +{ + "zookeeper": { + "connectionString": "127.0.0.1:2181/backbeat", + "autoCreateNamespace": false + }, + "kafka": { + "hosts": "127.0.0.1:9092", + "backlogMetrics": { + "zkPath": "/backbeat/run/kafka-backlog-metrics", + "intervalS": 60 + }, + "maxRequestSize": 5000020 + }, + "s3": { + "host": "127.0.0.1", + "port": 8000 + }, + "vaultAdmin": { + "host": "127.0.0.1", + "port": 8500 + }, + "replicationGroupId": "RG001 ", + "queuePopulator": { + "cronRule": "*/5 * * * * *", + "batchMaxRead": 10000, + "batchTimeoutMs": 9000, + "zookeeperPath": "/queue-populator", + "logSource": "mongo", + "bucketd": { + "host": "127.0.0.1", + "port": 9000 + }, + "dmd": { + "host": "127.0.0.1", + "port": 9990 + }, + "mongo": { + "replicaSetHosts": + "localhost:27017,localhost:27018,localhost:27019", + "writeConcern": "majority", + "replicaSet": "rs0", + "readPreference": "primary", + "database": "metadata" + }, + "kafka": { + "topic": "backbeat-oplog", + "consumerGroupId": "backbeat-qp-oplog-group" + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 4042 + } + }, + "extensions": { + "ingestion": { + "auth": { + "type": "service", + "account": "service-md-ingestion" + }, + "topic": "backbeat-ingestion", + "zookeeperPath": "/ingestion", + "cronRule": "*/5 * * * * *", + "maxParallelReaders": 5, + "sources": [ + { + "name": "zenko-bucket", + "bucket": "src-bucket", + "host": "localhost", + "port": 8000, + "https": false, + "type": "scality_s3", + "locationConstraint": "a-zenko-location", + "auth": { + "accessKey": "myAccessKey", + "secretKey": "myEncryptedSecretKey" + } + } + ], + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8550 + } + }, + "mongoProcessor": { + "topic": "backbeat-ingestion", + "groupId": "backbeat-mongo-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8551 + } + }, + "replication": { + "source": { + "transport": "http", + "s3": { + "host": "127.0.0.1", + "port": 8000 + }, + "auth": { + "type": "service", + "account": "service-replication", + "vault": { + "host": "127.0.0.1", + "port": 8500, + "adminPort": 8600 + } + } + }, + "destination": { + "transport": "http", + "bootstrapList": [ + { "site": "zenko", "servers": ["localhost:8001"], + "echo": false }, + { "site": "wontwork-location", "type": "aws_s3" }, + { "site": "aws-location", "type": "aws_s3" } + ], + "auth": { + "type": "service", + "account": "service-replication" + } + }, + "topic": "backbeat-replication", + "dataMoverTopic": "backbeat-data-mover", + "replicationStatusTopic": "backbeat-replication-status", + "replicationFailedTopic": "backbeat-replication-failed", + "monitorReplicationFailures": true, + "monitorReplicationFailureExpiryTimeS": 86400, + "replayTopics": [ + { + "topicName": "backbeat-replication-replay-0", + "retries": 5 + } + ], + "queueProcessor": { + "groupId": "backbeat-replication-group", + "retry": { + "aws_s3": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "azure": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "gcp": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "scality": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + } + }, + "concurrency": 10, + "mpuPartsConcurrency": 10, + "probeServer": { + "bindAddress": "localhost", + "port": 4043 + } + }, + "replicationStatusProcessor": { + "groupId": "backbeat-replication-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "localhost", + "port": 4045 + } + }, + "objectSizeMetrics": [ + 66560, + 8388608, + 68157440 + ] + }, + "lifecycle": { + "auth": { + "type": "service", + "account": "service-lifecycle" + }, + "zookeeperPath": "/lifecycle", + "bucketTasksTopic": "backbeat-lifecycle-bucket-tasks", + "objectTasksTopic": "backbeat-lifecycle-object-tasks", + "transitionTasksTopic": "backbeat-lifecycle-transition-tasks", + "conductor": { + "cronRule": "0 */5 * * * *", + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8552 + } + }, + "bucketProcessor": { + "groupId": "backbeat-lifecycle-bucket-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8553 + } + }, + "objectProcessor": { + "groupId": "backbeat-lifecycle-object-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8554 + } + }, + "coldStorageArchiveTopicPrefix": "cold-archive-req-", + "coldStorageRestoreTopicPrefix": "cold-restore-req-", + "coldStorageGCTopicPrefix": "cold-gc-req-", + "coldStorageStatusTopicPrefix": "cold-status-" + }, + "gc": { + "topic": "backbeat-gc", + "auth": { + "type": "service", + "account": "service-gc" + }, + "consumer": { + "groupId": "backbeat-gc-consumer-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10 + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8555 + } + }, + "oplogPopulator": { + "topic": "backbeat-oplog", + "kafkaConnectHost": "127.0.0.1", + "kafkaConnectPort": 8083, + "numberOfConnectors": 1, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8556 + } + }, + "notification": { + "topic": "backbeat-bucket-notification", + "monitorNotificationFailures": true, + "queueProcessor": { + "groupId": "backbeat-bucket-notification-group", + "concurrency": 10 + }, + "destinations": [ + { + "resource": "destination1", + "type": "kafka", + "host": "localhost", + "port": 9092, + "topic": "destination-topic-1", + "internalTopic": "internal-notification-topic-destination1", + "auth": {} + }, + { + "resource": "destination2", + "type": "kafka", + "host": "localhost:9092", + "port": 9092, + "topic": "destination-topic-2", + "internalTopic": "internal-notification-topic-destination2", + "auth": {} + } + ] + } + }, + "log": { + "logLevel": "info", + "dumpLevel": "error" + }, + "metrics": { + "topic": "backbeat-metrics" + }, + "server": { + "healthChecks": { + "allowFrom": ["127.0.0.1/8", "::1"] + }, + "host": "127.0.0.1", + "port": 8900 + }, + "redis": { + "host": "localhost", + "port": 6379 + }, + "certFilePaths": { + "key": "", + "cert": "", + "ca": "" + } +} From bb69b624b526918a4b6872cee5914ce3c085fcd9 Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 6 Nov 2024 13:16:19 +0100 Subject: [PATCH 5/5] remove unsupported config param healthcheckServer was removed from the config in BB-520 Issue: BB-623 --- tests/config.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/config.json b/tests/config.json index f3f6d8b39..4536aa80f 100644 --- a/tests/config.json +++ b/tests/config.json @@ -249,10 +249,6 @@ "host": "127.0.0.1", "port": 8900 }, - "healthcheckServer": { - "bindAddress": "0.0.0.0", - "port": 4042 - }, "redis": { "name": "backbeat-test", "password": "",