Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor RestApiTest and TestCase #17775

Merged
merged 3 commits into from
Jul 17, 2023
Merged

Refactor RestApiTest and TestCase #17775

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
84bea34
refactor RestApiTest and TestCase
007DXR Jul 14, 2023
ad0f36d
Update TestCase.java
007DXR Jul 17, 2023
109eaa1
remove useless sentence
007DXR Jul 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 25 additions & 65 deletions dora/tests/src/test/java/alluxio/client/rest/CreateBucketTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
import alluxio.Constants;
import alluxio.client.WriteType;
import alluxio.conf.PropertyKey;
import alluxio.master.journal.JournalType;
import alluxio.proxy.s3.S3Error;
import alluxio.proxy.s3.S3ErrorCode;
import alluxio.testutils.LocalAlluxioClusterResource;

Expand All @@ -25,45 +23,34 @@
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import org.gaul.s3proxy.junit.S3ProxyRule;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import java.net.HttpURLConnection;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;

public class CreateBucketTest extends RestApiTest {

private static final String TEST_BUCKET = "test-bucket";
private static final int UFS_PORT = 8002;
private AmazonS3 mS3Client = null;
@Rule
public S3ProxyRule mS3Proxy = S3ProxyRule.builder()
.withBlobStoreProvider("transient")
.withPort(8001)
.withPort(UFS_PORT)
.withCredentials("_", "_")
.build();

@Rule
public LocalAlluxioClusterResource mLocalAlluxioClusterResource =
new LocalAlluxioClusterResource.Builder()
.setIncludeProxy(true)
.setProperty(PropertyKey.MASTER_PERSISTENCE_CHECKER_INTERVAL_MS, "10ms")
.setProperty(PropertyKey.MASTER_PERSISTENCE_SCHEDULER_INTERVAL_MS, "10ms")
.setProperty(PropertyKey.JOB_MASTER_WORKER_HEARTBEAT_INTERVAL, "200ms")
.setProperty(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT, Constants.MB * 16)
.setProperty(PropertyKey.MASTER_TTL_CHECKER_INTERVAL_MS, Long.MAX_VALUE)
.setProperty(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.CACHE_THROUGH)
.setProperty(PropertyKey.USER_FILE_RESERVED_BYTES, Constants.MB * 16 / 2)
.setProperty(PropertyKey.CONF_DYNAMIC_UPDATE_ENABLED, true)
.setProperty(PropertyKey.WORKER_BLOCK_STORE_TYPE, "PAGE")
.setProperty(PropertyKey.WORKER_PAGE_STORE_PAGE_SIZE, Constants.KB)
.setProperty(PropertyKey.WORKER_PAGE_STORE_SIZES, "1GB")
.setProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.NOOP)
.setProperty(PropertyKey.UNDERFS_S3_ENDPOINT, "localhost:8001")
.setProperty(PropertyKey.UNDERFS_S3_ENDPOINT, "localhost:" + UFS_PORT)
.setProperty(PropertyKey.UNDERFS_S3_ENDPOINT_REGION, "us-west-2")
.setProperty(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS, true)
.setProperty(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS, "s3://" + TEST_BUCKET)
Expand All @@ -72,13 +59,10 @@ public class CreateBucketTest extends RestApiTest {
.setProperty(PropertyKey.S3A_ACCESS_KEY, mS3Proxy.getAccessKey())
.setProperty(PropertyKey.S3A_SECRET_KEY, mS3Proxy.getSecretKey())
.setNumWorkers(2)
.setStartCluster(false)
.build();

@Before
public void before() throws Exception {
mLocalAlluxioClusterResource.start();

mS3Client = AmazonS3ClientBuilder
.standard()
.withPathStyleAccessEnabled(true)
Expand All @@ -101,59 +85,35 @@ public void after() {
}

/**
* Creates a bucket.
*/
@Test
public void createAndHeadBucket() throws Exception {
String bucketName = "bucket";
Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(),
headBucketRestCall(bucketName).getResponseCode());
Assert.assertEquals(Response.Status.OK.getStatusCode(),
createBucketRestCall(bucketName).getResponseCode());
Assert.assertEquals(Response.Status.OK.getStatusCode(),
headBucketRestCall(bucketName).getResponseCode());
}

/**
* Creates an existent bucket.
* Creates a bucket. Creates an existent bucket.
*/
@Test
public void createExistentBucket() throws Exception {
public void createBucket() throws Exception {
String bucketName = "bucket";
// Ensures this bucket exists.
if (headBucketRestCall(bucketName).getResponseCode()
== Response.Status.NOT_FOUND.getStatusCode()) {
Assert.assertEquals(Response.Status.OK.getStatusCode(),
createBucketRestCall(bucketName).getResponseCode());
}

HttpURLConnection connection = createBucketRestCall(bucketName);
Assert.assertEquals(Response.Status.CONFLICT.getStatusCode(), connection.getResponseCode());
S3Error response =
new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream());
Assert.assertEquals(bucketName, response.getResource());
Assert.assertEquals(S3ErrorCode.Name.BUCKET_ALREADY_EXISTS, response.getCode());
// Heads a non-existent bucket.
headTestCase(bucketName).checkResponseCode(Status.NOT_FOUND.getStatusCode());
// Creates a bucket.
createBucketTestCase(bucketName).checkResponseCode(Status.OK.getStatusCode());
// Heads a bucket.
headTestCase(bucketName).checkResponseCode(Status.OK.getStatusCode());
// Creates an existent bucket.
createBucketTestCase(bucketName).checkResponseCode(Status.CONFLICT.getStatusCode())
.checkErrorCode(S3ErrorCode.Name.BUCKET_ALREADY_EXISTS);
}

/**
* 2 users creates the same bucket.
* Deletes a non-existent bucket. Deletes an existent bucket.
*/
@Test
public void createSameBucket() throws Exception {
public void deleteBucket() throws Exception {
String bucketName = "bucket";
// Ensures this bucket exists.
if (headBucketRestCall(bucketName, "user0").getResponseCode()
== Response.Status.NOT_FOUND.getStatusCode()) {
Assert.assertEquals(Response.Status.OK.getStatusCode(),
createBucketRestCall(bucketName, "user0").getResponseCode());
}

HttpURLConnection connection = createBucketRestCall(bucketName, "user1");

Assert.assertEquals(Response.Status.CONFLICT.getStatusCode(), connection.getResponseCode());
S3Error response =
new XmlMapper().readerFor(S3Error.class).readValue(connection.getErrorStream());
Assert.assertEquals(bucketName, response.getResource());
Assert.assertEquals(S3ErrorCode.Name.BUCKET_ALREADY_EXISTS, response.getCode());
headTestCase(bucketName).checkResponseCode(Status.NOT_FOUND.getStatusCode());
// Deletes a non-existent bucket.
deleteTestCase(bucketName).checkResponseCode(Status.NOT_FOUND.getStatusCode())
.checkErrorCode(S3ErrorCode.Name.NO_SUCH_BUCKET);
createBucketTestCase(bucketName).checkResponseCode(Status.OK.getStatusCode());
// Deletes an existent bucket.
deleteTestCase(bucketName).checkResponseCode(Status.NO_CONTENT.getStatusCode());
headTestCase(bucketName).checkResponseCode(Status.NOT_FOUND.getStatusCode());
}
}
3 changes: 1 addition & 2 deletions dora/tests/src/test/java/alluxio/client/rest/ListStatusTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -923,8 +923,7 @@ public void headAndListNonExistentBucket() throws Exception {
// Heads a non-existent bucket.
String bucketName = "non_existent_bucket";
// Verifies 404 status will be returned by head non-existent bucket.
Assert.assertEquals(Response.Status.NOT_FOUND.getStatusCode(),
headBucketRestCall(bucketName).getResponseCode());
headTestCase(bucketName).checkResponseCode(Response.Status.NOT_FOUND.getStatusCode());

// Lists objects in a non-existent bucket.
HttpURLConnection connection2 = new TestCase(mHostname, mPort, mBaseUri,
Expand Down
49 changes: 31 additions & 18 deletions dora/tests/src/test/java/alluxio/client/rest/RestApiTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@
import alluxio.testutils.BaseIntegrationTest;

import com.google.common.collect.ImmutableMap;
import com.google.common.io.BaseEncoding;

import java.net.HttpURLConnection;
import java.security.MessageDigest;
import java.util.Map;
import javax.validation.constraints.NotNull;
import javax.ws.rs.HttpMethod;
Expand All @@ -30,43 +31,55 @@ public abstract class RestApiTest extends BaseIntegrationTest {
protected int mPort;
protected String mBaseUri = Constants.REST_API_PREFIX;

protected HttpURLConnection createBucketRestCall(String bucket, @NotNull String user)
throws Exception {
return new TestCase(mHostname, mPort, mBaseUri,
bucket, NO_PARAMS, HttpMethod.PUT,
getDefaultOptionsWithAuth(user)).execute();
protected TestCase newTestCase(String bucket, Map<String, String> params,
String httpMethod, TestCaseOptions options) throws Exception {
return new TestCase(mHostname, mPort, mBaseUri, bucket, params, httpMethod,
options);
}

protected HttpURLConnection createBucketRestCall(String bucket) throws Exception {
return createBucketRestCall(bucket, TEST_USER_NAME);
protected TestCase createBucketTestCase(String bucket) throws Exception {
return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth());
}

protected HttpURLConnection headBucketRestCall(String bucket, @NotNull String user)
throws Exception {
return new TestCase(mHostname, mPort, mBaseUri,
bucket, NO_PARAMS, HttpMethod.HEAD,
getDefaultOptionsWithAuth(user)).execute();
protected TestCase createObjectTestCase(String bucket, byte[] object) throws Exception {
return newTestCase(bucket, NO_PARAMS, HttpMethod.PUT, getDefaultOptionsWithAuth()
.setBody(object)
.setMD5(computeObjectChecksum(object)));
}

protected TestCase deleteTestCase(String uri) throws Exception {
return newTestCase(uri, NO_PARAMS, HttpMethod.DELETE, getDefaultOptionsWithAuth());
}

protected HttpURLConnection headBucketRestCall(String bucket) throws Exception {
return headBucketRestCall(bucket, TEST_USER_NAME);
protected TestCase headTestCase(String uri) throws Exception {
return newTestCase(uri, NO_PARAMS, HttpMethod.HEAD, getDefaultOptionsWithAuth());
}

protected TestCase listTestCase(String uri, Map<String, String> params) throws Exception {
return newTestCase(uri, params, HttpMethod.GET,
getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE));
}

protected void listStatusRestCall(Map<String, String> parameters, ListBucketResult expected)
throws Exception {
new TestCase(mHostname, mPort, mBaseUri,
TEST_BUCKET_NAME, parameters, HttpMethod.GET,
getDefaultOptionsWithAuth())
getDefaultOptionsWithAuth().setContentType(TestCaseOptions.XML_CONTENT_TYPE))
.runAndCheckResult(expected);
}

protected TestCaseOptions getDefaultOptionsWithAuth(@NotNull String user) {
return TestCaseOptions.defaults()
.setAuthorization("AWS4-HMAC-SHA256 Credential=" + user + "/...")
.setContentType(TestCaseOptions.XML_CONTENT_TYPE);
.setAuthorization("AWS4-HMAC-SHA256 Credential=" + user + "/...");
}

protected TestCaseOptions getDefaultOptionsWithAuth() {
return getDefaultOptionsWithAuth(TEST_USER_NAME);
}

private String computeObjectChecksum(byte[] objectContent) throws Exception {
MessageDigest md5Hash = MessageDigest.getInstance("MD5");
byte[] md5Digest = md5Hash.digest(objectContent);
return BaseEncoding.base64().encode(md5Digest);
}
}
27 changes: 22 additions & 5 deletions dora/tests/src/test/java/alluxio/client/rest/S3ClientRestApiTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2205,12 +2205,29 @@ private void testTagging(String resource, ImmutableMap<String, String> expectedT
Assert.assertEquals(0, deletedTags.getTagMap().size());
}

private void createBucketRestCall(String bucketUri) throws Exception {
createBucketRestCall(bucketUri, TEST_USER_NAME);
}

private void createBucketRestCall(String bucketUri, String user) throws Exception {
TestCaseOptions options = getDefaultOptionsWithAuth(user);
new TestCase(mHostname, mPort, mBaseUri,
bucketUri, NO_PARAMS, HttpMethod.PUT,
options).runAndCheckResult();
}

private HttpURLConnection deleteBucketRestCall(String bucketUri) throws Exception {
return new TestCase(mHostname, mPort, mBaseUri,
bucketUri, NO_PARAMS, HttpMethod.DELETE,
getDefaultOptionsWithAuth()).executeAndAssertSuccess();
}

private HttpURLConnection headBucketRestCall(String bucketUri) throws Exception {
return new TestCase(mHostname, mPort, mBaseUri,
bucketUri, NO_PARAMS, HttpMethod.HEAD,
getDefaultOptionsWithAuth()).execute();
}

private String computeObjectChecksum(byte[] objectContent) throws Exception {
MessageDigest md5Hash = MessageDigest.getInstance("MD5");
byte[] md5Digest = md5Hash.digest(objectContent);
Expand All @@ -2236,13 +2253,13 @@ private String initiateMultipartUploadRestCall(String objectUri, String user) th
}

private TestCase getCompleteMultipartUploadReadCallTestCase(
String objectUri, String uploadId, CompleteMultipartUploadRequest request) {
String objectUri, String uploadId, CompleteMultipartUploadRequest request) throws Exception {
Map<String, String> params = ImmutableMap.of("uploadId", uploadId);
return new TestCase(mHostname, mPort, mBaseUri,
objectUri, params, HttpMethod.POST,
getDefaultOptionsWithAuth()
.setBody(request)
.setContentType(TestCaseOptions.XML_CONTENT_TYPE));
objectUri, params, HttpMethod.POST,
getDefaultOptionsWithAuth()
.setBody(request)
.setContentType(TestCaseOptions.XML_CONTENT_TYPE));
}

private String completeMultipartUploadRestCall(
Expand Down
Loading
Loading