Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add EC2 credential test for repository-s3 #31918

Merged
merged 15 commits into from
Jul 18, 2018
Merged

Add EC2 credential test for repository-s3 #31918

Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
0278c88
add EC2 credential test
Jul 10, 2018
a56d1cd
excluded ec2_credentials from minio tests
Jul 10, 2018
5fe4227
some gradle amendments on Alpar's review
Jul 10, 2018
30e86c8
changes due to David's review: use own bucket/basepath for EC2 test; …
Jul 10, 2018
46dded9
clean AmazonS3Fixture from hard-coded keys
Jul 10, 2018
ccfa0c5
added s3EC2ProfileName to gradle; added nonAuthPath / authPath to Ama…
Jul 11, 2018
c7af0b6
polish of AmazonS3Fixture
Jul 11, 2018
0d31887
Merge remote-tracking branch 'remotes/origin/master' into lazy_repo_init
Jul 12, 2018
fd28579
moved to properties file for s3Fixture
Jul 12, 2018
4006d19
Merge remote-tracking branch 'remotes/origin/master' into 26913_ec2_c…
Jul 12, 2018
a4b3613
fix s3FixtureProperties generation
Jul 12, 2018
9c4c3e7
make fixture random again
Jul 13, 2018
5430473
some code improvements
Jul 16, 2018
0ffae39
Merge remote-tracking branch 'remotes/origin/master' into 26913_ec2_c…
Jul 17, 2018
161deee
(hopefully) final polish of AmazonS3Fixture
Jul 17, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 53 additions & 52 deletions plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -84,11 +83,12 @@ private AmazonS3Fixture(final String workingDir, Properties properties) {
this.properties = properties;
this.random = new Random(Long.parseUnsignedLong(requireNonNull(properties.getProperty("tests.seed")), 16));

new Bucket("s3Fixture.permanent", false, null);
new Bucket("s3Fixture.temporary", true, null);
final Bucket ec2Bucket = new Bucket("s3Fixture.ec2", false, random);
new Bucket("s3Fixture.permanent", false);
new Bucket("s3Fixture.temporary", true);
final Bucket ec2Bucket = new Bucket("s3Fixture.ec2",
randomAsciiAlphanumOfLength(random, 10), randomAsciiAlphanumOfLength(random, 10));

this.handlers = defaultHandlers(buckets, ec2Bucket.name);
this.handlers = defaultHandlers(buckets, ec2Bucket);
}

private static String nonAuthPath(Request request) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

Expand Down Expand Up @@ -118,53 +118,49 @@ protected Response handle(final Request request) throws IOException {
final String authorizedPath = authPath(request);
final RequestHandler handler = handlers.retrieve(authorizedPath, request.getParameters());
if (handler != null) {
final String authorization = request.getHeader("Authorization");
if (authorization == null) {
final String bucketName = request.getParam("bucket");
if (bucketName == null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
}
final Collection<Bucket> values = buckets.values();
String permittedBucket = null;
for (final Bucket bucket : values) {
if (authorization.contains(bucket.key)) {
final String sessionToken = request.getHeader("x-amz-security-token");
if (bucket.token == null) {
if (sessionToken != null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", "");
}
} else {
if (sessionToken == null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
}
if (sessionToken.equals(bucket.token) == false) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
}
}
permittedBucket = bucket.name;
break;
}
final Bucket bucket = buckets.get(bucketName);
if (bucket == null) {
return newBucketNotFoundError(request.getId(), bucketName);
}

if (permittedBucket == null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
final Response authResponse = authenticateBucket(request, bucket);
if (authResponse != null) {
return authResponse;
}

final String bucket = request.getParam("bucket");
if (bucket != null && permittedBucket.equals(bucket) == false) {
// allow a null bucket to support the multi-object-delete API which
// passes the bucket name in the host header instead of the URL.
if (buckets.containsKey(bucket)) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", "");
} else {
return newBucketNotFoundError(request.getId(), bucket);
}
}
return handler.handle(request);

} else {
return newInternalError(request.getId(), "No handler defined for request [" + request + "]");
}
}

private Response authenticateBucket(Request request, Bucket bucket) {
final String authorization = request.getHeader("Authorization");
if (authorization == null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", "");
}
if (authorization.contains(bucket.key)) {
final String sessionToken = request.getHeader("x-amz-security-token");
if (bucket.token == null) {
if (sessionToken != null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", "");
}
} else {
if (sessionToken == null) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", "");
}
if (sessionToken.equals(bucket.token) == false) {
return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", "");
}
}
}
return null;
}

public static void main(final String[] args) throws Exception {
if (args == null || args.length != 2) {
throw new IllegalArgumentException("AmazonS3Fixture <working directory> <property file>");
Expand All @@ -178,7 +174,7 @@ public static void main(final String[] args) throws Exception {
}

/** Builds the default request handlers **/
private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final String ec2BucketName) {
private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> buckets, final Bucket ec2Bucket) {
final PathTrie<RequestHandler> handlers = new PathTrie<>(RestUtils.REST_DECODER);

// HEAD Object
Expand Down Expand Up @@ -325,7 +321,7 @@ private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> bucke
// Delete Multiple Objects
//
// https://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
handlers.insert(authPath(HttpPost.METHOD_NAME, "/"), (request) -> {
handlers.insert(nonAuthPath(HttpPost.METHOD_NAME, "/"), (request) -> {
final List<String> deletes = new ArrayList<>();
final List<String> errors = new ArrayList<>();

Expand All @@ -348,7 +344,12 @@ private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> bucke

boolean found = false;
for (Bucket bucket : buckets.values()) {
if (bucket.objects.remove(objectName) != null) {
if (bucket.objects.containsKey(objectName)) {
final Response authResponse = authenticateBucket(request, bucket);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ooh nice.

if (authResponse != null) {
return authResponse;
}
bucket.objects.remove(objectName);
found = true;
}
}
Expand Down Expand Up @@ -393,7 +394,6 @@ private PathTrie<RequestHandler> defaultHandlers(final Map<String, Bucket> bucke
return new Response(RestStatus.OK.getStatus(), headers, response.getBytes(UTF_8));
});

final Bucket ec2Bucket = buckets.get(ec2BucketName);
// GET
//
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
Expand Down Expand Up @@ -428,15 +428,16 @@ class Bucket {
/** Blobs contained in the bucket **/
final Map<String, byte[]> objects;

private Bucket(final String prefix, final boolean tokenRequired, final Random random) {
private Bucket(final String prefix, final boolean tokenRequired) {
this(prefix, prop(properties, prefix + "_key"),
tokenRequired ? prop(properties, prefix + "_session_token") : null);
}

private Bucket(final String prefix, final String key, final String token) {
this.name = prop(properties, prefix + "_bucket_name");
if (random != null) {
this.key = randomAsciiAlphanumOfLength(random, 10);
this.token = randomAsciiAlphanumOfLength(random, 10);
} else {
this.key = prop(properties, prefix + "_key");
this.token = tokenRequired ? prop(properties, prefix + "_session_token") : null;
}
this.key = key;
this.token = token;

this.objects = ConcurrentCollections.newConcurrentMap();
if (buckets.put(name, this) != null) {
throw new IllegalArgumentException("bucket " + name + " is already registered");
Expand Down