Skip to content

Commit

Permalink
feat(client-s3): support generating endpoints from multi-region acces…
Browse files Browse the repository at this point in the history 
…s point (#2796)

* Revert "fix(client-s3): revert MRAP customizations (#2759)"

This reverts commit cfb3fff.

* chore: update mrap packages dependencies

* feat(middleware-sdk-s3): dynamically import aws-crt package optional dependencies

* feat(middleware-user-agent): track if aws-crt is available at runtime

* chore: update crt signer loading error message

* chore(middleware-sdk-s3): add crt package to dev dependency and track usage

This makes sure the type of CRT package is available and the Lerna builds
packages in right order--build crt signer package before buiding s3 middleware
package.

* docs(client-s3): add docs to Bucket members on using MRAP

* chore(signature-v4-crt): update crt version to ^1.9.7

* fix(client-s3): address PR feedbacks
  • Loading branch information
@AllanZhengYP
AllanZhengYP authored Sep 20, 2021
1 parent 66de569 commit c1bed9d
Show file tree
Hide file tree
Showing 51 changed files with 2,750 additions and 260 deletions.
168 changes: 168 additions & 0 deletions clients/client-s3/models/models_0.ts
View file

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions clients/client-s3/models/models_1.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,8 @@ export interface RestoreObjectRequest {
* <p>The bucket name containing the object to restore. </p>
* <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p>
* <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com. When using this action using S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html">Using S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>
* <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
* For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
*/
Bucket: string | undefined;

Expand Down Expand Up @@ -898,6 +900,8 @@ export namespace ScanRange {
export interface SelectObjectContentRequest {
/**
* <p>The S3 bucket.</p>
* <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
* For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
*/
Bucket: string | undefined;

Expand Down Expand Up @@ -1055,6 +1059,8 @@ export interface UploadPartRequest {
* <p>The name of the bucket to which the multipart upload was initiated.</p>
* <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p>
* <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com. When using this action using S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html">Using S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>
* <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
* For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
*/
Bucket: string | undefined;

Expand Down Expand Up @@ -1221,6 +1227,8 @@ export interface UploadPartCopyRequest {
* <p>The bucket name.</p>
* <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p>
* <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com. When using this action using S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html">Using S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>
* <p>Note: To supply the Multi-region Access Point (MRAP) to Bucket, you need to install the "@aws-sdk/signature-v4-crt" package to your project dependencies.
* For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>
*/
Bucket: string | undefined;

Expand Down
2 changes: 2 additions & 0 deletions clients/client-s3/runtimeConfig.shared.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { defaultRegionInfoProvider } from "./endpoints";
import { S3SignatureV4 } from "@aws-sdk/middleware-sdk-s3";
import { Logger as __Logger } from "@aws-sdk/types";
import { parseUrl } from "@aws-sdk/url-parser";
import { S3ClientConfig } from "./S3Client";
Expand All @@ -12,6 +13,7 @@ export const getRuntimeConfig = (config: S3ClientConfig) => ({
logger: config?.logger ?? ({} as __Logger),
regionInfoProvider: config?.regionInfoProvider ?? defaultRegionInfoProvider,
serviceId: config?.serviceId ?? "S3",
signerConstructor: config?.signerConstructor ?? S3SignatureV4,
signingEscapePath: config?.signingEscapePath ?? false,
urlParser: config?.urlParser ?? parseUrl,
useArnRegion: config?.useArnRegion ?? false,
Expand Down
49 changes: 49 additions & 0 deletions ...ript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,24 @@
import static software.amazon.smithy.typescript.codegen.integration.RuntimeClientPlugin.Convention.HAS_MIDDLEWARE;

import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.logging.Logger;
import software.amazon.smithy.aws.traits.ServiceTrait;
import software.amazon.smithy.build.PluginContext;
import software.amazon.smithy.codegen.core.SymbolProvider;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.knowledge.OperationIndex;
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.OperationShape;
import software.amazon.smithy.model.shapes.ServiceShape;
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.shapes.ShapeId;
import software.amazon.smithy.model.shapes.StructureShape;
import software.amazon.smithy.model.traits.DocumentationTrait;
import software.amazon.smithy.typescript.codegen.LanguageTarget;
import software.amazon.smithy.typescript.codegen.TypeScriptDependency;
import software.amazon.smithy.typescript.codegen.TypeScriptSettings;
Expand All @@ -45,6 +53,7 @@
*/
@SmithyInternalApi
public final class AddS3Config implements TypeScriptIntegration {
private static final Logger LOGGER = Logger.getLogger(AddS3Config.class.getName());

private static final Set<String> SSEC_OPERATIONS = SetUtils.of("SSECustomerKey", "CopySourceSSECustomerKey");

Expand All @@ -60,6 +69,42 @@ public final class AddS3Config implements TypeScriptIntegration {
"CompleteMultipartUpload"
);

private static final String CRT_NOTIFICATION = "<p>Note: To supply the Multi-region Access Point (MRAP) to Bucket,"
+ " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n"
+ "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues</p>";

@Override
public Model preprocessModel(PluginContext context, TypeScriptSettings settings) {
Model model = context.getModel();
ServiceShape serviceShape = settings.getService(model);
if (!testServiceId(serviceShape)) {
return model;
}
Model.Builder modelBuilder = model.toBuilder();
Set<StructureShape> inputShapes = new HashSet<>();
for (ShapeId operationId : serviceShape.getAllOperations()) {
OperationShape operationShape = model.expectShape(operationId, OperationShape.class);
if (NON_BUCKET_ENDPOINT_OPERATIONS.contains(operationShape.getId().getName(serviceShape))) {
continue;
}
operationShape.getInput().ifPresent(inputShapeId -> {
StructureShape inputShape = model.expectShape(inputShapeId, StructureShape.class);
inputShape.getMember("Bucket").ifPresent(bucketMember -> {
bucketMember.getTrait(DocumentationTrait.class).ifPresent(documentationTrait -> {
StructureShape.Builder inputShapeBuilder = inputShape.toBuilder();
MemberShape.Builder builder = MemberShape.shapeToBuilder(bucketMember);
String newDocString = documentationTrait.getValue() + "\n" + CRT_NOTIFICATION;
MemberShape newMemberShape = builder.addTrait(new DocumentationTrait(newDocString)).build();
inputShapeBuilder.addMember(newMemberShape);
inputShapes.add(inputShapeBuilder.build());
});
});
});
}
LOGGER.info("Patching " + inputShapes.size() + " input shapes with CRT notification");
return modelBuilder.addShapes(inputShapes).build();
}

@Override
public void addConfigInterfaceFields(TypeScriptSettings settings, Model model, SymbolProvider symbolProvider,
TypeScriptWriter writer) {
Expand Down Expand Up @@ -87,6 +132,10 @@ public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(TypeScrip
writer.write("false");
}, "useArnRegion", writer -> {
writer.write("false");
}, "signerConstructor", writer -> {
writer.addDependency(AwsDependency.S3_MIDDLEWARE)
.addImport("S3SignatureV4", "S3SignatureV4", AwsDependency.S3_MIDDLEWARE.packageName)
.write("S3SignatureV4");
});
case NODE:
return MapUtils.of("useArnRegion", writer -> {
Expand Down
1 change: 1 addition & 0 deletions packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ describe("bucketEndpointMiddleware", () => {
clientSigningRegion: mockRegion,
useArnRegion: false,
isCustomEndpoint: false,
disableMultiregionAccessPoints: false,
});
expect(previouslyResolvedConfig.region).toBeCalled();
expect(previouslyResolvedConfig.regionInfoProvider).toBeCalled();
Expand Down
131 changes: 63 additions & 68 deletions packages/middleware-bucket-endpoint/src/bucketEndpointMiddleware.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,81 +15,76 @@ import { bucketHostname } from "./bucketHostname";
import { getPseudoRegion } from "./bucketHostnameUtils";
import { BucketEndpointResolvedConfig } from "./configurations";

export const bucketEndpointMiddleware =
(options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> =>
<Output extends MetadataBearer>(
next: BuildHandler<any, Output>,
context: HandlerExecutionContext
): BuildHandler<any, Output> =>
async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
const { Bucket: bucketName } = args.input as { Bucket: string };
let replaceBucketInPath = options.bucketEndpoint;
const request = args.request;
if (HttpRequest.isInstance(request)) {
if (options.bucketEndpoint) {
request.hostname = bucketName;
} else if (validateArn(bucketName)) {
const bucketArn = parseArn(bucketName);
const clientRegion = getPseudoRegion(await options.region());
const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
const useArnRegion = await options.useArnRegion();
const {
hostname,
bucketEndpoint,
signingRegion: modifiedSigningRegion,
signingService,
} = bucketHostname({
bucketName: bucketArn,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
useArnRegion,
clientPartition: partition,
clientSigningRegion: signingRegion,
clientRegion: clientRegion,
isCustomEndpoint: options.isCustomEndpoint,
});
export const bucketEndpointMiddleware = (options: BucketEndpointResolvedConfig): BuildMiddleware<any, any> => <
Output extends MetadataBearer
>(
next: BuildHandler<any, Output>,
context: HandlerExecutionContext
): BuildHandler<any, Output> => async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
const { Bucket: bucketName } = args.input as { Bucket: string };
let replaceBucketInPath = options.bucketEndpoint;
const request = args.request;
if (HttpRequest.isInstance(request)) {
if (options.bucketEndpoint) {
request.hostname = bucketName;
} else if (validateArn(bucketName)) {
const bucketArn = parseArn(bucketName);
const clientRegion = getPseudoRegion(await options.region());
const { partition, signingRegion = clientRegion } = (await options.regionInfoProvider(clientRegion)) || {};
const useArnRegion = await options.useArnRegion();
const { hostname, bucketEndpoint, signingRegion: modifiedSigningRegion, signingService } = bucketHostname({
bucketName: bucketArn,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
useArnRegion,
clientPartition: partition,
clientSigningRegion: signingRegion,
clientRegion: clientRegion,
isCustomEndpoint: options.isCustomEndpoint,
disableMultiregionAccessPoints: await options.disableMultiregionAccessPoints(),
});

// If the request needs to use a region or service name inferred from ARN that different from client region, we
// need to set them in the handler context so the signer will use them
if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
context["signing_region"] = modifiedSigningRegion;
}
if (signingService && signingService !== "s3") {
context["signing_service"] = signingService;
}
// If the request needs to use a region or service name inferred from ARN that different from client region, we
// need to set them in the handler context so the signer will use them
if (modifiedSigningRegion && modifiedSigningRegion !== signingRegion) {
context["signing_region"] = modifiedSigningRegion;
}
if (signingService && signingService !== "s3") {
context["signing_service"] = signingService;
}

request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
} else {
const clientRegion = getPseudoRegion(await options.region());
const { hostname, bucketEndpoint } = bucketHostname({
bucketName,
clientRegion,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
isCustomEndpoint: options.isCustomEndpoint,
});
request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
} else {
const clientRegion = getPseudoRegion(await options.region());
const { hostname, bucketEndpoint } = bucketHostname({
bucketName,
clientRegion,
baseHostname: request.hostname,
accelerateEndpoint: options.useAccelerateEndpoint,
dualstackEndpoint: options.useDualstackEndpoint,
pathStyleEndpoint: options.forcePathStyle,
tlsCompatible: request.protocol === "https:",
isCustomEndpoint: options.isCustomEndpoint,
});

request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
}
request.hostname = hostname;
replaceBucketInPath = bucketEndpoint;
}

if (replaceBucketInPath) {
request.path = request.path.replace(/^(\/)?[^\/]+/, "");
if (request.path === "") {
request.path = "/";
}
if (replaceBucketInPath) {
request.path = request.path.replace(/^(\/)?[^\/]+/, "");
if (request.path === "") {
request.path = "/";
}
}
}

return next({ ...args, request });
};
return next({ ...args, request });
};

export const bucketEndpointMiddlewareOptions: RelativeMiddlewareOptions = {
tags: ["BUCKET_ENDPOINT"],
Expand Down
Loading

0 comments on commit c1bed9d

Please sign in to comment.