Skip to content

Commit

Permalink
listen to oplog when a transition to cold workflow is disabled
Browse files Browse the repository at this point in the history 
This is needed to keep restore operations on cold objects working

Issue: BB-457
  • Loading branch information
@Kerkesni
Kerkesni committed Oct 23, 2023
1 parent 02d0b64 commit 33c4186
Show file tree
Hide file tree
Showing 3 changed files with 185 additions and 64 deletions.
79 changes: 63 additions & 16 deletions extensions/lifecycle/LifecycleOplogPopulatorUtils.js
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,60 @@
const OplogPopulatorUtils = require('../../lib/util/OplogPopulatorUtils');
const locations = require('../../conf/locationConfig.json');

class LifecycleOplogPopulatorUtils extends OplogPopulatorUtils {

/**
* Get extension specific MongoDB filter
* to get buckets that have the lifecycle
* extension enabled
* @returns {Object} MongoDB filter
*/
static getExtensionMongoDBFilter() {
// getting buckets with at least one lifecycle
// rule enabled
const coldLocations = Object.keys(locations).filter(loc => locations[loc].isCold);
// getting buckets with at least one lifecycle transition rule enabled
// or buckets with disabled lifecycle transition rules to cold locations (needed for restores)
return {
'value.lifecycleConfiguration.rules': {
$elemMatch: {
ruleStatus: 'Enabled',
actions: {
$elemMatch: {
$and: [
{
actions: {
$elemMatch: {
actionName: {
$in: ['Transition', 'NoncurrentVersionTransition'],
},
},
},
}, {
$or: [
{ actionName: 'Transition' },
{ actionName: 'NoncurrentVersionTransition' },
{
ruleStatus: 'Enabled'
},
{
actions: {
transition: {
$elemMatch: {
storageClass: {
$in: coldLocations
}
},
},
},
},
{
actions: {
nonCurrentVersionTransition: {
$elemMatch: {
storageClass: {
$in: coldLocations
}
},
},
}
},
],
},
},
],
},
}
};
Expand All @@ -40,14 +73,28 @@ class LifecycleOplogPopulatorUtils extends OplogPopulatorUtils {
if (!rules || rules.length === 0) {
return false;
} else {
// return true if at least one lifecycle
// rule is enabled
return rules.some(rule =>
rule.ruleStatus === 'Enabled' &&
rule.actions.some(
action => ['Transition', 'NoncurrentVersionTransition'].includes(action.actionName)
)
);
// return true if at least one lifecycle transition rule is enabled
// or if buckets have transition rules to cold locations (needed for restores)
return rules.some(rule => {
if (rule.ruleStatus === 'Enabled') {
return rule.actions.some(
action => ['Transition', 'NoncurrentVersionTransition']
.includes(action.actionName)
);
}
return rule.actions.some(action => {
if (action.actionName === 'Transition') {
return action.transition.some(
transition => locations[transition.storageClass]?.isCold
);
} else if (action.actionName === 'NoncurrentVersionTransition') {
return action.nonCurrentVersionTransition.some(
transition => locations[transition.storageClass]?.isCold
);
}
return false;
});
});
}
}
}
Expand Down
58 changes: 56 additions & 2 deletions tests/unit/lifecycle/LifecycleOplogPopulatorUtils.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,17 @@ const currentTransitionRule = {
transition: [
{
days: 0,
storageClass: 'dmf',
storageClass: 'some-location',
},
],
};

const currentTransitionRuleCold = {
actionName: 'Transition',
transition: [
{
days: 0,
storageClass: 'location-dmf-v1',
},
],
};
Expand All @@ -16,7 +26,17 @@ const nonCurrentTransitionRule = {
nonCurrentVersionTransition: [
{
noncurrentDays: 0,
storageClass: 'dmf',
storageClass: 'some-location',
},
],
};

const nonCurrentTransitionRuleCold = {
actionName: 'NoncurrentVersionTransition',
nonCurrentVersionTransition: [
{
noncurrentDays: 0,
storageClass: 'location-dmf-v1',
},
],
};
Expand Down Expand Up @@ -152,6 +172,40 @@ describe('LifecycleOplogPopulatorUtils', () => {
},
expectedReturn: false,
},
{
it: 'should return true when transition rule is disabled for a cold location (current)',
bucketMd: {
lifecycleConfiguration: {
rules: [
buildLifecycleRule([
currentTransitionRuleCold,
], false),
buildLifecycleRule([
currentTransitionRule,
nonCurrentTransitionRule,
], false),
],
},
},
expectedReturn: true,
},
{
it: 'should return true when transition rule is disabled for a cold location (non current)',
bucketMd: {
lifecycleConfiguration: {
rules: [
buildLifecycleRule([
nonCurrentTransitionRuleCold,
], false),
buildLifecycleRule([
currentTransitionRule,
nonCurrentTransitionRule,
], false),
],
},
},
expectedReturn: true,
},
].forEach(params => {
it(params.it, () => {
const enabled = LifecycleOplogPopulatorUtils.isBucketExtensionEnabled(params.bucketMd);
Expand Down
112 changes: 66 additions & 46 deletions tests/unit/oplogPopulator/oplogPopulator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ const { MongoClient } = require('mongodb');

const logger = new werelogs.Logger('connect-wrapper-logger');

const locations = require('../../../conf/locationConfig.json');

const OplogPopulator =
require('../../../extensions/oplogPopulator/OplogPopulator');
const ChangeStream =
Expand Down Expand Up @@ -208,71 +210,89 @@ describe('OplogPopulator', () => {
});

describe('_getBackbeatEnabledBuckets', () => {
const coldLocations = Object.keys(locations).filter(loc => locations[loc].isCold);
const replicationFilter = {
'value.replicationConfiguration.rules': {
$elemMatch: {
enabled: true,
},
},
};
const notificationFilter = {
'value.notificationConfiguration': {
$type: 3,
},
};
const lifecycleFilter = {
'value.lifecycleConfiguration.rules': {
$elemMatch: {
$and: [
{
actions: {
$elemMatch: {
actionName: {
$in: ['Transition', 'NoncurrentVersionTransition'],
},
},
},
}, {
$or: [
{
ruleStatus: 'Enabled'
},
{
actions: {
transition: {
$elemMatch: {
storageClass: {
$in: coldLocations
}
},
},
},
},
{
actions: {
nonCurrentVersionTransition: {
$elemMatch: {
storageClass: {
$in: coldLocations
}
},
},
}
},
],
},
],
},
},
};
[
{
extensions: ['notification'],
filter: [
{ 'value.notificationConfiguration': { $type: 3 } },
notificationFilter,
],
},
{
extensions: ['replication'],
filter: [
{
'value.replicationConfiguration.rules': {
$elemMatch: {
enabled: true,
},
},
},
replicationFilter
],
},
{
extensions: ['lifecycle'],
filter: [
{
'value.lifecycleConfiguration.rules': {
$elemMatch: {
ruleStatus: 'Enabled',
actions: {
$elemMatch: {
$or: [
{ actionName: 'Transition' },
{ actionName: 'NoncurrentVersionTransition' },
],
},
},
},
},
},
lifecycleFilter,
],
},
{
extensions: ['notification', 'replication', 'lifecycle'],
filter: [
{ 'value.notificationConfiguration': { $type: 3 } },
{
'value.replicationConfiguration.rules': {
$elemMatch: {
enabled: true,
},
},
},
{
'value.lifecycleConfiguration.rules': {
$elemMatch: {
ruleStatus: 'Enabled',
actions: {
$elemMatch: {
$or: [
{ actionName: 'Transition' },
{ actionName: 'NoncurrentVersionTransition' },
],
},
},
},
},
},
notificationFilter,
replicationFilter,
lifecycleFilter,
],
}
].forEach(scenario => {
Expand Down

0 comments on commit 33c4186

Please sign in to comment.