diff --git a/extensions/notification/NotificationConfigValidator.js b/extensions/notification/NotificationConfigValidator.js index fd7bc690e..924f89e94 100644 --- a/extensions/notification/NotificationConfigValidator.js +++ b/extensions/notification/NotificationConfigValidator.js @@ -33,7 +33,9 @@ const joiSchema = joi.object({ concurrency: joi.number().greater(0).default(1000), }, destinations: joi.array().items(destinationSchema).default([]), - probeServer: probeServerJoi.default() + // TODO: BB-625 reset to being required after supporting probeserver in S3C + // for bucket notification proceses + probeServer: probeServerJoi.optional() }); function configValidator(backbeatConfig, extConfig) { diff --git a/lib/Config.js b/lib/Config.js index 4860a0dbe..290560576 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -2,6 +2,7 @@ const assert = require('assert'); const { EventEmitter } = require('events'); +const joi = require('joi'); const fs = require('fs'); const path = require('path'); @@ -54,10 +55,16 @@ class Config extends EventEmitter { throw new Error(`could not parse config file: ${err.message}`); } - const { value: parsedConfig, err } = backbeatConfigJoi.validate(config); - if (err) { - throw new Error(`could not validate config file: ${err.message}`); - } + this._parseConfig(config); + } + + /** + * Parses Backbeat's configuration + * @param {Object} config backbeat configuration + * @returns {undefined} + */ + _parseConfig(config) { + const parsedConfig = joi.attempt(config, backbeatConfigJoi); if (parsedConfig.extensions) { Object.keys(parsedConfig.extensions).forEach(extName => { diff --git a/lib/config.joi.js b/lib/config.joi.js index a83d92a80..18a89862e 100644 --- a/lib/config.joi.js +++ b/lib/config.joi.js @@ -55,7 +55,13 @@ const joiSchema = joi.object({ }).when('logSource', { is: 'dmd', then: joi.required() }), mongo: mongoJoi, kafka: qpKafkaJoi.when('logSource', { is: 'kafka', then: joi.required() }), - probeServer: probeServerJoi.default(), + // TODO: BB-625 reset to being required after supporting probeserver in S3C + // for bucket notification proceses + probeServer: probeServerJoi.when('...extensions', { + is: joi.object().keys({ notification: joi.exist() }), + then: joi.optional(), + otherwise: joi.required(), + }), circuitBreaker: joi.object().optional(), }, log: logJoi, diff --git a/lib/util/probe.js b/lib/util/probe.js index 2274122a0..d214db424 100644 --- a/lib/util/probe.js +++ b/lib/util/probe.js @@ -26,8 +26,7 @@ const RdkafkaStats = require('node-rdkafka-prometheus'); */ function startProbeServer(config, callback) { if (!config) { - const err = new Error('configuration for probe server is missing'); - callback(err); + callback(); return; } diff --git a/tests/config.json b/tests/config.json index f3f6d8b39..4536aa80f 100644 --- a/tests/config.json +++ b/tests/config.json @@ -249,10 +249,6 @@ "host": "127.0.0.1", "port": 8900 }, - "healthcheckServer": { - "bindAddress": "0.0.0.0", - "port": 4042 - }, "redis": { "name": "backbeat-test", "password": "", diff --git a/tests/unit/lib/config/Config.spec.js b/tests/unit/lib/config/Config.spec.js new file mode 100644 index 000000000..f3160d20a --- /dev/null +++ b/tests/unit/lib/config/Config.spec.js @@ -0,0 +1,25 @@ +'use strict'; // eslint-disable-line + +const assert = require('assert'); + +const { Config } = require('../../../../lib/Config'); +const backbeatConfig = require('./config.json'); + +describe('Config', () => { + it('should make the probeserver config in the queuePoulator' + + 'required when multiple extensions are configured', () => { + const config = new Config(); + const testConfig = { ...backbeatConfig }; + delete testConfig.queuePopulator.probeServer; + assert.throws(() => config._parseConfig(testConfig)); + }); + + it('should make the probeserver config in the queuePoulator' + + 'optional when only notification config is specified', () => { + const config = new Config(); + const testConfig = { ...backbeatConfig }; + delete testConfig.queuePopulator.probeServer; + testConfig.extensions = { notification: testConfig.extensions.notification }; + assert.doesNotThrow(() => config._parseConfig(testConfig)); + }); +}); diff --git a/tests/unit/lib/config/config.json b/tests/unit/lib/config/config.json new file mode 100644 index 000000000..57b697014 --- /dev/null +++ b/tests/unit/lib/config/config.json @@ -0,0 +1,364 @@ +{ + "zookeeper": { + "connectionString": "127.0.0.1:2181/backbeat", + "autoCreateNamespace": false + }, + "kafka": { + "hosts": "127.0.0.1:9092", + "backlogMetrics": { + "zkPath": "/backbeat/run/kafka-backlog-metrics", + "intervalS": 60 + }, + "maxRequestSize": 5000020 + }, + "s3": { + "host": "127.0.0.1", + "port": 8000 + }, + "vaultAdmin": { + "host": "127.0.0.1", + "port": 8500 + }, + "replicationGroupId": "RG001 ", + "queuePopulator": { + "cronRule": "*/5 * * * * *", + "batchMaxRead": 10000, + "batchTimeoutMs": 9000, + "zookeeperPath": "/queue-populator", + "logSource": "mongo", + "bucketd": { + "host": "127.0.0.1", + "port": 9000 + }, + "dmd": { + "host": "127.0.0.1", + "port": 9990 + }, + "mongo": { + "replicaSetHosts": + "localhost:27017,localhost:27018,localhost:27019", + "writeConcern": "majority", + "replicaSet": "rs0", + "readPreference": "primary", + "database": "metadata" + }, + "kafka": { + "topic": "backbeat-oplog", + "consumerGroupId": "backbeat-qp-oplog-group" + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 4042 + } + }, + "extensions": { + "ingestion": { + "auth": { + "type": "service", + "account": "service-md-ingestion" + }, + "topic": "backbeat-ingestion", + "zookeeperPath": "/ingestion", + "cronRule": "*/5 * * * * *", + "maxParallelReaders": 5, + "sources": [ + { + "name": "zenko-bucket", + "bucket": "src-bucket", + "host": "localhost", + "port": 8000, + "https": false, + "type": "scality_s3", + "locationConstraint": "a-zenko-location", + "auth": { + "accessKey": "myAccessKey", + "secretKey": "myEncryptedSecretKey" + } + } + ], + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8550 + } + }, + "mongoProcessor": { + "topic": "backbeat-ingestion", + "groupId": "backbeat-mongo-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8551 + } + }, + "replication": { + "source": { + "transport": "http", + "s3": { + "host": "127.0.0.1", + "port": 8000 + }, + "auth": { + "type": "service", + "account": "service-replication", + "vault": { + "host": "127.0.0.1", + "port": 8500, + "adminPort": 8600 + } + } + }, + "destination": { + "transport": "http", + "bootstrapList": [ + { "site": "zenko", "servers": ["localhost:8001"], + "echo": false }, + { "site": "wontwork-location", "type": "aws_s3" }, + { "site": "aws-location", "type": "aws_s3" } + ], + "auth": { + "type": "service", + "account": "service-replication" + } + }, + "topic": "backbeat-replication", + "dataMoverTopic": "backbeat-data-mover", + "replicationStatusTopic": "backbeat-replication-status", + "replicationFailedTopic": "backbeat-replication-failed", + "monitorReplicationFailures": true, + "monitorReplicationFailureExpiryTimeS": 86400, + "replayTopics": [ + { + "topicName": "backbeat-replication-replay-0", + "retries": 5 + } + ], + "queueProcessor": { + "groupId": "backbeat-replication-group", + "retry": { + "aws_s3": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "azure": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "gcp": { + "maxRetries": 5, + "timeoutS": 900, + "backoff": { + "min": 60000, + "max": 900000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "scality": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + } + }, + "concurrency": 10, + "mpuPartsConcurrency": 10, + "probeServer": { + "bindAddress": "localhost", + "port": 4043 + } + }, + "replicationStatusProcessor": { + "groupId": "backbeat-replication-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "localhost", + "port": 4045 + } + }, + "objectSizeMetrics": [ + 66560, + 8388608, + 68157440 + ] + }, + "lifecycle": { + "auth": { + "type": "service", + "account": "service-lifecycle" + }, + "zookeeperPath": "/lifecycle", + "bucketTasksTopic": "backbeat-lifecycle-bucket-tasks", + "objectTasksTopic": "backbeat-lifecycle-object-tasks", + "transitionTasksTopic": "backbeat-lifecycle-transition-tasks", + "conductor": { + "cronRule": "0 */5 * * * *", + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8552 + } + }, + "bucketProcessor": { + "groupId": "backbeat-lifecycle-bucket-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8553 + } + }, + "objectProcessor": { + "groupId": "backbeat-lifecycle-object-processor-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8554 + } + }, + "coldStorageArchiveTopicPrefix": "cold-archive-req-", + "coldStorageRestoreTopicPrefix": "cold-restore-req-", + "coldStorageGCTopicPrefix": "cold-gc-req-", + "coldStorageStatusTopicPrefix": "cold-status-" + }, + "gc": { + "topic": "backbeat-gc", + "auth": { + "type": "service", + "account": "service-gc" + }, + "consumer": { + "groupId": "backbeat-gc-consumer-group", + "retry": { + "maxRetries": 5, + "timeoutS": 300, + "backoff": { + "min": 1000, + "max": 300000, + "jitter": 0.1, + "factor": 1.5 + } + }, + "concurrency": 10 + }, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8555 + } + }, + "oplogPopulator": { + "topic": "backbeat-oplog", + "kafkaConnectHost": "127.0.0.1", + "kafkaConnectPort": 8083, + "numberOfConnectors": 1, + "probeServer": { + "bindAddress": "0.0.0.0", + "port": 8556 + } + }, + "notification": { + "topic": "backbeat-bucket-notification", + "monitorNotificationFailures": true, + "queueProcessor": { + "groupId": "backbeat-bucket-notification-group", + "concurrency": 10 + }, + "destinations": [ + { + "resource": "destination1", + "type": "kafka", + "host": "localhost", + "port": 9092, + "topic": "destination-topic-1", + "internalTopic": "internal-notification-topic-destination1", + "auth": {} + }, + { + "resource": "destination2", + "type": "kafka", + "host": "localhost:9092", + "port": 9092, + "topic": "destination-topic-2", + "internalTopic": "internal-notification-topic-destination2", + "auth": {} + } + ] + } + }, + "log": { + "logLevel": "info", + "dumpLevel": "error" + }, + "metrics": { + "topic": "backbeat-metrics" + }, + "server": { + "healthChecks": { + "allowFrom": ["127.0.0.1/8", "::1"] + }, + "host": "127.0.0.1", + "port": 8900 + }, + "redis": { + "host": "localhost", + "port": 6379 + }, + "certFilePaths": { + "key": "", + "cert": "", + "ca": "" + } +} diff --git a/tests/unit/lib/util/probe.spec.js b/tests/unit/lib/util/probe.spec.js index 133d816c5..fd893431d 100644 --- a/tests/unit/lib/util/probe.spec.js +++ b/tests/unit/lib/util/probe.spec.js @@ -4,10 +4,10 @@ const { startProbeServer, getReplicationProbeConfig } = const Logger = require('werelogs').Logger; describe('Probe server', () => { - it('is not created with no config', done => { + it('should return undefined when no config is passed', done => { const config = undefined; startProbeServer(config, (err, probeServer) => { - assert(err); + assert.ifError(err); assert.strictEqual(probeServer, undefined); done(); });