diff --git a/extensions/lifecycle/LifecycleOplogPopulatorUtils.js b/extensions/lifecycle/LifecycleOplogPopulatorUtils.js index fb2a28403..3e5b79c7f 100644 --- a/extensions/lifecycle/LifecycleOplogPopulatorUtils.js +++ b/extensions/lifecycle/LifecycleOplogPopulatorUtils.js @@ -1,6 +1,8 @@ const OplogPopulatorUtils = require('../../lib/util/OplogPopulatorUtils'); +const locations = require('../../conf/locationConfig.json'); class LifecycleOplogPopulatorUtils extends OplogPopulatorUtils { + /** * Get extension specific MongoDB filter * to get buckets that have the lifecycle @@ -8,20 +10,51 @@ class LifecycleOplogPopulatorUtils extends OplogPopulatorUtils { * @returns {Object} MongoDB filter */ static getExtensionMongoDBFilter() { - // getting buckets with at least one lifecycle - // rule enabled + const coldLocations = Object.keys(locations).filter(loc => locations[loc].isCold); + // getting buckets with at least one lifecycle transition rule enabled + // or buckets with disabled lifecycle transition rules to cold locations (needed for restores) return { 'value.lifecycleConfiguration.rules': { $elemMatch: { - ruleStatus: 'Enabled', - actions: { - $elemMatch: { + $and: [ + { + actions: { + $elemMatch: { + actionName: { + $in: ['Transition', 'NoncurrentVersionTransition'], + }, + }, + }, + }, { $or: [ - { actionName: 'Transition' }, - { actionName: 'NoncurrentVersionTransition' }, + { + ruleStatus: 'Enabled' + }, + { + actions: { + transition: { + $elemMatch: { + storageClass: { + $in: coldLocations + } + }, + }, + }, + }, + { + actions: { + nonCurrentVersionTransition: { + $elemMatch: { + storageClass: { + $in: coldLocations + } + }, + }, + } + }, ], }, - }, + ], }, } }; @@ -40,14 +73,28 @@ class LifecycleOplogPopulatorUtils extends OplogPopulatorUtils { if (!rules || rules.length === 0) { return false; } else { - // return true if at least one lifecycle - // rule is enabled - return rules.some(rule => - rule.ruleStatus === 'Enabled' && - rule.actions.some( - action => ['Transition', 'NoncurrentVersionTransition'].includes(action.actionName) - ) - ); + // return true if at least one lifecycle transition rule is enabled + // or if buckets have transition rules to cold locations (needed for restores) + return rules.some(rule => { + if (rule.ruleStatus === 'Enabled') { + return rule.actions.some( + action => ['Transition', 'NoncurrentVersionTransition'] + .includes(action.actionName) + ); + } + return rule.actions.some(action => { + if (action.actionName === 'Transition') { + return action.transition.some( + transition => locations[transition.storageClass]?.isCold + ); + } else if (action.actionName === 'NoncurrentVersionTransition') { + return action.nonCurrentVersionTransition.some( + transition => locations[transition.storageClass]?.isCold + ); + } + return false; + }); + }); } } } diff --git a/tests/unit/lifecycle/LifecycleOplogPopulatorUtils.spec.js b/tests/unit/lifecycle/LifecycleOplogPopulatorUtils.spec.js index 98af0f82d..47a643667 100644 --- a/tests/unit/lifecycle/LifecycleOplogPopulatorUtils.spec.js +++ b/tests/unit/lifecycle/LifecycleOplogPopulatorUtils.spec.js @@ -6,7 +6,17 @@ const currentTransitionRule = { transition: [ { days: 0, - storageClass: 'dmf', + storageClass: 'some-location', + }, + ], +}; + +const currentTransitionRuleCold = { + actionName: 'Transition', + transition: [ + { + days: 0, + storageClass: 'location-dmf-v1', }, ], }; @@ -16,7 +26,17 @@ const nonCurrentTransitionRule = { nonCurrentVersionTransition: [ { noncurrentDays: 0, - storageClass: 'dmf', + storageClass: 'some-location', + }, + ], +}; + +const nonCurrentTransitionRuleCold = { + actionName: 'NoncurrentVersionTransition', + nonCurrentVersionTransition: [ + { + noncurrentDays: 0, + storageClass: 'location-dmf-v1', }, ], }; @@ -152,6 +172,40 @@ describe('LifecycleOplogPopulatorUtils', () => { }, expectedReturn: false, }, + { + it: 'should return true when transition rule is disabled for a cold location (current)', + bucketMd: { + lifecycleConfiguration: { + rules: [ + buildLifecycleRule([ + currentTransitionRuleCold, + ], false), + buildLifecycleRule([ + currentTransitionRule, + nonCurrentTransitionRule, + ], false), + ], + }, + }, + expectedReturn: true, + }, + { + it: 'should return true when transition rule is disabled for a cold location (non current)', + bucketMd: { + lifecycleConfiguration: { + rules: [ + buildLifecycleRule([ + nonCurrentTransitionRuleCold, + ], false), + buildLifecycleRule([ + currentTransitionRule, + nonCurrentTransitionRule, + ], false), + ], + }, + }, + expectedReturn: true, + }, ].forEach(params => { it(params.it, () => { const enabled = LifecycleOplogPopulatorUtils.isBucketExtensionEnabled(params.bucketMd); diff --git a/tests/unit/oplogPopulator/oplogPopulator.js b/tests/unit/oplogPopulator/oplogPopulator.js index 3a387938f..f25c6729f 100644 --- a/tests/unit/oplogPopulator/oplogPopulator.js +++ b/tests/unit/oplogPopulator/oplogPopulator.js @@ -7,6 +7,8 @@ const { MongoClient } = require('mongodb'); const logger = new werelogs.Logger('connect-wrapper-logger'); +const locations = require('../../../conf/locationConfig.json'); + const OplogPopulator = require('../../../extensions/oplogPopulator/OplogPopulator'); const ChangeStream = @@ -208,71 +210,89 @@ describe('OplogPopulator', () => { }); describe('_getBackbeatEnabledBuckets', () => { + const coldLocations = Object.keys(locations).filter(loc => locations[loc].isCold); + const replicationFilter = { + 'value.replicationConfiguration.rules': { + $elemMatch: { + enabled: true, + }, + }, + }; + const notificationFilter = { + 'value.notificationConfiguration': { + $type: 3, + }, + }; + const lifecycleFilter = { + 'value.lifecycleConfiguration.rules': { + $elemMatch: { + $and: [ + { + actions: { + $elemMatch: { + actionName: { + $in: ['Transition', 'NoncurrentVersionTransition'], + }, + }, + }, + }, { + $or: [ + { + ruleStatus: 'Enabled' + }, + { + actions: { + transition: { + $elemMatch: { + storageClass: { + $in: coldLocations + } + }, + }, + }, + }, + { + actions: { + nonCurrentVersionTransition: { + $elemMatch: { + storageClass: { + $in: coldLocations + } + }, + }, + } + }, + ], + }, + ], + }, + }, + }; [ { extensions: ['notification'], filter: [ - { 'value.notificationConfiguration': { $type: 3 } }, + notificationFilter, ], }, { extensions: ['replication'], filter: [ - { - 'value.replicationConfiguration.rules': { - $elemMatch: { - enabled: true, - }, - }, - }, + replicationFilter ], }, { extensions: ['lifecycle'], filter: [ - { - 'value.lifecycleConfiguration.rules': { - $elemMatch: { - ruleStatus: 'Enabled', - actions: { - $elemMatch: { - $or: [ - { actionName: 'Transition' }, - { actionName: 'NoncurrentVersionTransition' }, - ], - }, - }, - }, - }, - }, + lifecycleFilter, ], }, { extensions: ['notification', 'replication', 'lifecycle'], filter: [ - { 'value.notificationConfiguration': { $type: 3 } }, - { - 'value.replicationConfiguration.rules': { - $elemMatch: { - enabled: true, - }, - }, - }, - { - 'value.lifecycleConfiguration.rules': { - $elemMatch: { - ruleStatus: 'Enabled', - actions: { - $elemMatch: { - $or: [ - { actionName: 'Transition' }, - { actionName: 'NoncurrentVersionTransition' }, - ], - }, - }, - }, - }, - }, + notificationFilter, + replicationFilter, + lifecycleFilter, ], } ].forEach(scenario => {