Skip to content

Commit

Permalink
Merge pull request #435 from yahoo/s3
Browse files Browse the repository at this point in the history 
reuse s3 client during initial load
  • Loading branch information
@mujibur
mujibur authored Mar 28, 2018
2 parents 82403ee + 75f7b58 commit d01cc47
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 33 deletions.
37 changes: 34 additions & 3 deletions servers/zts/src/main/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class S3ChangeLogStore implements ChangeLogStore {
long lastModTime = 0;
CloudStore cloudStore = null;
String s3BucketName = null;
AmazonS3 awsS3Client = null;

public S3ChangeLogStore(CloudStore cloudStore) {
this.cloudStore = cloudStore;
Expand All @@ -63,8 +64,24 @@ public boolean supportsFullRefresh() {

@Override
public SignedDomain getSignedDomain(String domainName) {
AmazonS3 s3 = getS3Client();
return getSignedDomain(s3, domainName);

// make sure we have an aws s3 client for our request

if (awsS3Client == null) {
awsS3Client = getS3Client();
}

SignedDomain signedDomain = getSignedDomain(awsS3Client, domainName);

// if we got a failure for any reason, we're going
// get a new aws s3 cilent and try again

if (signedDomain == null) {
awsS3Client = getS3Client();
signedDomain = getSignedDomain(awsS3Client, domainName);
}

return signedDomain;
}

SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {
Expand Down Expand Up @@ -182,14 +199,25 @@ public List<String> getLocalDomainList() {
lastModTime = System.currentTimeMillis();
}

// we are going to initialize our s3 client here since
// this is the first entry point before we start
// fetching all the domains individually

awsS3Client = getS3Client();

ArrayList<String> domains = new ArrayList<>();
listObjects(getS3Client(), domains, 0);
listObjects(awsS3Client, domains, 0);
return domains;
}

@Override
public Set<String> getServerDomainList() {

// for the server domain list operation since it's called
// periodically by the thread to see if any domains have
// been deleted, we're going to get a new s3 client
// insetad of using our original client

HashSet<String> domains = new HashSet<>();
listObjects(getS3Client(), domains, 0);
return domains;
Expand All @@ -212,6 +240,9 @@ public SignedDomains getUpdatedSignedDomains(StringBuilder lastModTimeBuffer) {
// based on its last modification timestamp so we need to get
// the full list and filter ourselves

// instead of using our fetched s3 client, we're going to
// obtain a new one to get the changes

AmazonS3 s3 = getS3Client();
ArrayList<String> domains = new ArrayList<>();
listObjects(s3, domains, lastModTime);
Expand Down
7 changes: 2 additions & 5 deletions servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/MockS3ChangeLogStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,14 @@
import com.yahoo.athenz.zts.store.impl.S3ChangeLogStore;

public class MockS3ChangeLogStore extends S3ChangeLogStore {

AmazonS3 s3 = null;

public MockS3ChangeLogStore(CloudStore cloudStore) {
super(cloudStore);
s3 = mock(AmazonS3.class);
awsS3Client = mock(AmazonS3.class);
}

@Override
AmazonS3 getS3Client() {
return s3;
return awsS3Client;
}

}
72 changes: 47 additions & 25 deletions servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,10 @@ public void testListObjectsAllObjectsNoPage() {
ObjectListing objectListing = mock(ObjectListing.class);
when(objectListing.getObjectSummaries()).thenReturn(objectList);
when(objectListing.isTruncated()).thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);

ArrayList<String> domains = new ArrayList<>();
store.listObjects(store.s3, domains, 0);
store.listObjects(store.awsS3Client, domains, 0);

assertEquals(domains.size(), 2);
assertTrue(domains.contains("iaas"));
Expand All @@ -125,10 +125,10 @@ public void testListObjectsAllObjectsNoPageModTime() {
ObjectListing objectListing = mock(ObjectListing.class);
when(objectListing.getObjectSummaries()).thenReturn(objectList);
when(objectListing.isTruncated()).thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);

ArrayList<String> domains = new ArrayList<>();
store.listObjects(store.s3, domains, (new Date(150)).getTime());
store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime());

assertEquals(domains.size(), 1);
assertTrue(domains.contains("iaas.athenz"));
Expand Down Expand Up @@ -172,11 +172,11 @@ public void testListObjectsAllObjectsMultiplePages() {
.thenReturn(true)
.thenReturn(true)
.thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.s3.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);

ArrayList<String> domains = new ArrayList<>();
store.listObjects(store.s3, domains, 0);
store.listObjects(store.awsS3Client, domains, 0);

assertEquals(domains.size(), 6);
assertTrue(domains.contains("iaas"));
Expand Down Expand Up @@ -231,11 +231,11 @@ public void testListObjectsAllObjectsMultiplePagesModTime() {
.thenReturn(true)
.thenReturn(true)
.thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.s3.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing);

ArrayList<String> domains = new ArrayList<>();
store.listObjects(store.s3, domains, (new Date(150)).getTime());
store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime());

assertEquals(domains.size(), 3);
assertTrue(domains.contains("cd.docker"));
Expand All @@ -258,7 +258,7 @@ public void testGetLocalDomains() {
ObjectListing objectListing = mock(ObjectListing.class);
when(objectListing.getObjectSummaries()).thenReturn(objectList);
when(objectListing.isTruncated()).thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);

// verify that our last mod time is 0 before the call

Expand Down Expand Up @@ -292,7 +292,7 @@ public void testGetServerDomains() {
ObjectListing objectListing = mock(ObjectListing.class);
when(objectListing.getObjectSummaries()).thenReturn(objectList);
when(objectListing.isTruncated()).thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);

// verify that our last mod time is 0 before the call

Expand All @@ -314,25 +314,25 @@ public void testGetServerDomains() {
@Test
public void testGetSignedDomainNotFound() {
MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
when(store.s3.getObject(any(GetObjectRequest.class))).thenReturn((S3Object) null);
when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenReturn((S3Object) null);

assertNull(store.getSignedDomain(store.s3, "iaas"));
assertNull(store.getSignedDomain(store.awsS3Client, "iaas"));
}

@Test
public void testGetSignedDomainClientException() {
MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);

when(store.s3.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonClientException("failed client operation"));
assertNull(store.getSignedDomain(store.s3, "iaas"));
when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonClientException("failed client operation"));
assertNull(store.getSignedDomain(store.awsS3Client, "iaas"));
}

@Test
public void testGetSignedDomainServiceException() {
MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);

when(store.s3.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonServiceException("failed server operation"));
assertNull(store.getSignedDomain(store.s3, "iaas"));
when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonServiceException("failed server operation"));
assertNull(store.getSignedDomain(store.awsS3Client, "iaas"));
}

private class MockS3ObjectInputStream extends S3ObjectInputStream {
Expand All @@ -351,9 +351,9 @@ public void testGetSignedDomainInternal() throws IOException {
S3Object object = mock(S3Object.class);
when(object.getObjectContent()).thenReturn(s3Is);

when(store.s3.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);
when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);

SignedDomain signedDomain = store.getSignedDomain(store.s3, "iaas");
SignedDomain signedDomain = store.getSignedDomain(store.awsS3Client, "iaas");
assertNotNull(signedDomain);
DomainData domainData = signedDomain.getDomain();
assertNotNull(domainData);
Expand All @@ -371,7 +371,29 @@ public void testGetSignedDomain() throws IOException {
S3Object object = mock(S3Object.class);
when(object.getObjectContent()).thenReturn(s3Is);

when(store.s3.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);
when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);

SignedDomain signedDomain = store.getSignedDomain("iaas");
assertNotNull(signedDomain);
DomainData domainData = signedDomain.getDomain();
assertNotNull(domainData);
assertEquals(domainData.getName(), "iaas");
is.close();
}

@Test
public void testGetSignedDomainException() throws IOException {
MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);

InputStream is = new FileInputStream("src/test/resources/iaas.json");
MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null);

S3Object object = mock(S3Object.class);
when(object.getObjectContent()).thenReturn(s3Is);

// first call we return null, second call we return success

when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenThrow(new AmazonServiceException("test")).thenReturn(object);

SignedDomain signedDomain = store.getSignedDomain("iaas");
assertNotNull(signedDomain);
Expand Down Expand Up @@ -399,7 +421,7 @@ public void testGetUpdatedSignedDomainsNoChanges() {
ObjectListing objectListing = mock(ObjectListing.class);
when(objectListing.getObjectSummaries()).thenReturn(objectList);
when(objectListing.isTruncated()).thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);

// set the last modification time to not return any of the domains
store.lastModTime = (new Date(250)).getTime();
Expand Down Expand Up @@ -428,16 +450,16 @@ public void testGetUpdatedSignedDomainsWithChange() throws FileNotFoundException
ObjectListing objectListing = mock(ObjectListing.class);
when(objectListing.getObjectSummaries()).thenReturn(objectList);
when(objectListing.isTruncated()).thenReturn(false);
when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);

InputStream is = new FileInputStream("src/test/resources/iaas.json");
MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null);

S3Object object = mock(S3Object.class);
when(object.getObjectContent()).thenReturn(s3Is);

when(store.s3.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);
when(store.s3.getObject("athenz-domain-sys.auth", "iaas.athenz")).thenReturn(object);
when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object);
when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas.athenz")).thenReturn(object);

// set the last modification time to return one of the domains
store.lastModTime = (new Date(150)).getTime();
Expand Down

0 comments on commit d01cc47

Please sign in to comment.