diff --git a/.gitignore b/.gitignore index 93b66809884..a5f9bb0f2c2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.d.ts node_modules cdk.context.json +.DS_Store +lib/.DS_Store # CDK asset staging directory .cdk.staging diff --git a/README.md b/README.md index f9b1f351ade..236aa1c9345 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ - [Required context parameters](#required-context-parameters) - [Interacting with OpenSearch cluster](#interacting-with-opensearch-cluster) - [Restricting Server Access](#restricting-server-access) + - [Enable Remote Store Feature](#enable-remote-store-feature) - [Check Logs](#check-logs) - [Access EC2 Instances](#access-ec2-instances) - [Port Mapping](#port-mapping) @@ -62,6 +63,7 @@ In order to deploy both the stacks the user needs to provide a set of required a | mlNodeStorage (Optional) | string | User provided ebs block storage size, defaults to 100Gb | | use50PercentHeap (Optional) | boolean | Boolean flag to use 50% of physical memory as heap. Default is 1GB. e.g., `--context use50PercentHeap=true` | | isInternal (Optional) | boolean | Boolean flag to make network load balancer internal. Default is internet-facing e.g., `--context isInternal=true` | +| enableRemoteStore (Optional) | boolean | Boolean flag to enable Remote Store feature e.g., `--context enableRemoteStore=true`. See [Enable Remote Store Feature](#enable-remote-store-feature) for more details. | @@ -131,6 +133,19 @@ Below values are allowed: | prefixList | Prefix List id (eg: ab-12345) | | securityGroupId | A security group ID (eg: sg-123456789) | +### Enable Remote Store Feature + +`Remote Store` feature provides an option to store indexed data in a remote durable data store. To enable this feature the user needs to register a snapshot repository (S3 or File System) which is used to store the index data. +Apart from passing `enableRemoteStore` flag as `true` the user needs to be provide additional settings to `opensearch.yml`, the settings are: +``` +1. opensearch.experimental.feature.remote_store.enabled: 'true' +2. cluster.remote_store.enabled: 'true' +3. opensearch.experimental.feature.segment_replication_experimental.enabled: 'true' +4. cluster.indices.replication.strategy: SEGMENT +``` +The above-mentioned settings need to be passed using `additionalConfig` parameter. +Please note the `experimental` settings are only applicable till the feature is under development and will be removed when the feature becomes GA. + ## Check logs The opensearch logs are available in cloudwatch logs log-group `opensearchLogGroup/opensearch.log` in the same region your stack is deployed. diff --git a/lib/infra/infra-stack.ts b/lib/infra/infra-stack.ts index 5cb6489e30f..4d2f693b27d 100644 --- a/lib/infra/infra-stack.ts +++ b/lib/infra/infra-stack.ts @@ -34,6 +34,7 @@ import { dump, load } from 'js-yaml'; import { InstanceTarget } from 'aws-cdk-lib/aws-elasticloadbalancingv2-targets'; import { CloudwatchAgent } from '../cloudwatch/cloudwatch-agent'; import { nodeConfig } from '../opensearch-config/node-config'; +import { RemoteStoreResources } from './remote-store-resources'; export interface infraProps extends StackProps{ readonly vpc: IVpc, @@ -59,9 +60,12 @@ export interface infraProps extends StackProps{ readonly mlEc2InstanceType: InstanceType, readonly use50PercentHeap: boolean, readonly isInternal: boolean, + readonly enableRemoteStore: boolean } export class InfraStack extends Stack { + private instanceRole: Role; + constructor(scope: Stack, id: string, props: infraProps) { super(scope, id, props); let opensearchListener: NetworkListener; @@ -79,13 +83,20 @@ export class InfraStack extends Stack { removalPolicy: RemovalPolicy.DESTROY, }); - const instanceRole = new Role(this, 'instanceRole', { + this.instanceRole = new Role(this, 'instanceRole', { managedPolicies: [ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ReadOnlyAccess'), ManagedPolicy.fromAwsManagedPolicyName('CloudWatchAgentServerPolicy'), ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')], assumedBy: new ServicePrincipal('ec2.amazonaws.com'), }); + if (props.enableRemoteStore) { + // Remote Store needs an S3 bucket to be registered as snapshot repo + // Add scoped bucket policy to the instance role attached to the EC2 + const remoteStoreObj = new RemoteStoreResources(this); + this.instanceRole.addToPolicy(remoteStoreObj.getRemoteStoreBucketPolicy()); + } + const singleNodeInstanceType = (props.cpuType === AmazonLinuxCpuType.X86_64) ? InstanceType.of(InstanceClass.R5, InstanceSize.XLARGE) : InstanceType.of(InstanceClass.R6G, InstanceSize.XLARGE); @@ -126,7 +137,7 @@ export class InfraStack extends Stack { generation: AmazonLinuxGeneration.AMAZON_LINUX_2, cpuType: props.cpuType, }), - role: instanceRole, + role: this.instanceRole, vpcSubnets: { subnetType: SubnetType.PRIVATE_WITH_EGRESS, }, @@ -174,7 +185,7 @@ export class InfraStack extends Stack { generation: AmazonLinuxGeneration.AMAZON_LINUX_2, cpuType: props.cpuType, }), - role: instanceRole, + role: this.instanceRole, maxCapacity: managerAsgCapacity, minCapacity: managerAsgCapacity, desiredCapacity: managerAsgCapacity, @@ -206,7 +217,7 @@ export class InfraStack extends Stack { generation: AmazonLinuxGeneration.AMAZON_LINUX_2, cpuType: props.cpuType, }), - role: instanceRole, + role: this.instanceRole, maxCapacity: 1, minCapacity: 1, desiredCapacity: 1, @@ -234,7 +245,7 @@ export class InfraStack extends Stack { generation: AmazonLinuxGeneration.AMAZON_LINUX_2, cpuType: props.cpuType, }), - role: instanceRole, + role: this.instanceRole, maxCapacity: dataAsgCapacity, minCapacity: dataAsgCapacity, desiredCapacity: dataAsgCapacity, @@ -264,7 +275,7 @@ export class InfraStack extends Stack { generation: AmazonLinuxGeneration.AMAZON_LINUX_2, cpuType: props.cpuType, }), - role: instanceRole, + role: this.instanceRole, maxCapacity: props.clientNodeCount, minCapacity: props.clientNodeCount, desiredCapacity: props.clientNodeCount, @@ -295,7 +306,7 @@ export class InfraStack extends Stack { generation: AmazonLinuxGeneration.AMAZON_LINUX_2, cpuType: props.cpuType, }), - role: instanceRole, + role: this.instanceRole, maxCapacity: props.mlNodeCount, minCapacity: props.mlNodeCount, desiredCapacity: props.mlNodeCount, @@ -445,14 +456,36 @@ export class InfraStack extends Stack { } if (props.distributionUrl.includes('artifacts.opensearch.org') && !props.minDistribution) { - cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch; echo "y"|sudo -u ec2-user bin/opensearch-plugin install discovery-ec2', { + cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install discovery-ec2 --batch', { cwd: '/home/ec2-user', ignoreErrors: false, })); } else { - cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch; echo "y"|sudo -u ec2-user bin/opensearch-plugin install ' + cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install ' + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}` - + `/tar/builds/opensearch/core-plugins/discovery-ec2-${props.opensearchVersion}.zip`, { + + `/tar/builds/opensearch/core-plugins/discovery-ec2-${props.opensearchVersion}.zip --batch`, { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); + } + + if (props.enableRemoteStore) { + if (props.distributionUrl.includes('artifacts.opensearch.org') && !props.minDistribution) { + cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install repository-s3 --batch', { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); + } else { + cfnInitConfig.push(InitCommand.shellCommand('set -ex;cd opensearch;sudo -u ec2-user bin/opensearch-plugin install ' + + `https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${props.opensearchVersion}/latest/linux/${props.cpuArch}` + + `/tar/builds/opensearch/core-plugins/repository-s3-${props.opensearchVersion}.zip --batch`, { + cwd: '/home/ec2-user', + ignoreErrors: false, + })); + } + + // eslint-disable-next-line max-len + cfnInitConfig.push(InitCommand.shellCommand(`set -ex;cd opensearch; echo "cluster.remote_store.repository: ${scope.stackName}-repo" >> config/opensearch.yml`, { cwd: '/home/ec2-user', ignoreErrors: false, })); @@ -519,6 +552,35 @@ export class InfraStack extends Stack { })); } + if (props.enableRemoteStore) { + // Snapshot creation call should be done from one node to avoid any race condition, using seed node. + if (nodeType === 'seed-manager' || nodeType === 'seed-data') { + if (props.securityDisabled) { + // eslint-disable-next-line max-len + cfnInitConfig.push(InitCommand.shellCommand(`set -ex; sleep 60; curl -XPUT "http://localhost:9200/_snapshot/${scope.stackName}-repo" -H 'Content-Type: application/json' -d' + { + "type": "s3", + "settings": { + "bucket": "${scope.stackName}", + "region": "${scope.region}", + "base_path": "remote-store" + } + }'`)); + } else { + // eslint-disable-next-line max-len + cfnInitConfig.push(InitCommand.shellCommand(`set -ex; sleep 60; curl -XPUT "https://localhost:9200/_snapshot/${scope.stackName}-repo" -ku admin:admin -H 'Content-Type: application/json' -d' + { + "type": "s3", + "settings": { + "bucket": "${scope.stackName}", + "region": "${scope.region}", + "base_path": "remote-store" + } + }'`)); + } + } + } + // If OSD Url is present if (props.dashboardsUrl !== 'undefined') { cfnInitConfig.push(InitCommand.shellCommand(`set -ex;mkdir opensearch-dashboards; curl -L ${props.dashboardsUrl} -o opensearch-dashboards.tar.gz;` diff --git a/lib/infra/remote-store-resources.ts b/lib/infra/remote-store-resources.ts new file mode 100644 index 00000000000..7c8a2486301 --- /dev/null +++ b/lib/infra/remote-store-resources.ts @@ -0,0 +1,37 @@ +import { RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { Effect, Policy, PolicyStatement } from 'aws-cdk-lib/aws-iam'; + +export class RemoteStoreResources { + private readonly snapshotS3Bucket: Bucket + + private readonly bucketPolicyStatement: PolicyStatement + + constructor(scope: Stack) { + this.snapshotS3Bucket = new Bucket(scope, `remote-store-${scope.stackName}`, { + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + bucketName: `${scope.stackName}`, + }); + + this.bucketPolicyStatement = new PolicyStatement({ + effect: Effect.ALLOW, + actions: [ + 's3:ListBucket', + 's3:GetBucketLocation', + 's3:ListBucketMultipartUploads', + 's3:ListBucketVersions', + 's3:GetObject', + 's3:PutObject', + 's3:DeleteObject', + 's3:AbortMultipartUpload', + 's3:ListMultipartUploadParts', + ], + resources: [this.snapshotS3Bucket.bucketArn, `${this.snapshotS3Bucket.bucketArn}/*`], + }); + } + + public getRemoteStoreBucketPolicy() { + return this.bucketPolicyStatement; + } +} diff --git a/lib/os-cluster-entrypoint.ts b/lib/os-cluster-entrypoint.ts index 638918b5ca4..7ff541fd019 100644 --- a/lib/os-cluster-entrypoint.ts +++ b/lib/os-cluster-entrypoint.ts @@ -182,6 +182,9 @@ export class OsClusterEntrypoint { const nlbScheme = `${scope.node.tryGetContext('isInternal')}`; const isInternal = nlbScheme === 'true'; + const remoteStore = `${scope.node.tryGetContext('enableRemoteStore')}`; + const enableRemoteStore = remoteStore === 'true'; + const network = new NetworkStack(scope, 'opensearch-network-stack', { cidrBlock: cidrRange, maxAzs: 3, @@ -230,6 +233,7 @@ export class OsClusterEntrypoint { additionalConfig: ymlConfig, use50PercentHeap, isInternal, + enableRemoteStore, ...props, }); diff --git a/test/os-cluster.test.ts b/test/os-cluster.test.ts index 0bf47f79c83..f4b5110014d 100644 --- a/test/os-cluster.test.ts +++ b/test/os-cluster.test.ts @@ -305,3 +305,92 @@ test('Test multi-node cluster with only data-nodes', () => { ], }); }); + +test('Test multi-node cluster with remote-store enabled', () => { + const app = new App({ + context: { + securityDisabled: true, + minDistribution: false, + distributionUrl: 'www.example.com', + cpuArch: 'x64', + singleNodeCluster: false, + dashboardsUrl: 'www.example.com', + distVersion: '1.0.0', + serverAccessType: 'ipv4', + restrictServerAccessTo: 'all', + managerNodeCount: 0, + dataNodeCount: 3, + dataNodeStorage: 200, + enableRemoteStore: true, + }, + }); + + // WHEN + const testStack = new OsClusterEntrypoint(app, { + env: { account: 'test-account', region: 'us-east-1' }, + }); + expect(testStack.stacks).toHaveLength(2); + + const infraStack = testStack.stacks.filter((s) => s.stackName === 'opensearch-infra-stack')[0]; + const infraTemplate = Template.fromStack(infraStack); + infraTemplate.resourceCountIs('AWS::S3::Bucket', 1); + infraTemplate.resourceCountIs('AWS::S3::BucketPolicy', 1); + infraTemplate.resourceCountIs('AWS::Lambda::Function', 1); + infraTemplate.resourceCountIs('AWS::IAM::Role', 2); + infraTemplate.resourceCountIs('AWS::IAM::Policy', 1); + infraTemplate.hasResourceProperties('AWS::S3::Bucket', { + BucketName: 'opensearch-infra-stack', + }); + infraTemplate.hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:ListBucket', + 's3:GetBucketLocation', + 's3:ListBucketMultipartUploads', + 's3:ListBucketVersions', + 's3:GetObject', + 's3:PutObject', + 's3:DeleteObject', + 's3:AbortMultipartUpload', + 's3:ListMultipartUploadParts', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'remotestoreopensearchinfrastack6A47755C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'remotestoreopensearchinfrastack6A47755C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + { + Action: [ + 'cloudformation:DescribeStackResource', + 'cloudformation:SignalResource', + ], + Effect: 'Allow', + Resource: { + Ref: 'AWS::StackId', + }, + }, + ], + }, + }); +});