diff --git a/servers/zts/src/main/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStore.java b/servers/zts/src/main/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStore.java index a513aa25954..462c60bc8c2 100644 --- a/servers/zts/src/main/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStore.java +++ b/servers/zts/src/main/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStore.java @@ -46,6 +46,7 @@ public class S3ChangeLogStore implements ChangeLogStore { long lastModTime = 0; CloudStore cloudStore = null; String s3BucketName = null; + AmazonS3 awsS3Client = null; public S3ChangeLogStore(CloudStore cloudStore) { this.cloudStore = cloudStore; @@ -63,8 +64,24 @@ public boolean supportsFullRefresh() { @Override public SignedDomain getSignedDomain(String domainName) { - AmazonS3 s3 = getS3Client(); - return getSignedDomain(s3, domainName); + + // make sure we have an aws s3 client for our request + + if (awsS3Client == null) { + awsS3Client = getS3Client(); + } + + SignedDomain signedDomain = getSignedDomain(awsS3Client, domainName); + + // if we got a failure for any reason, we're going + // get a new aws s3 cilent and try again + + if (signedDomain == null) { + awsS3Client = getS3Client(); + signedDomain = getSignedDomain(awsS3Client, domainName); + } + + return signedDomain; } SignedDomain getSignedDomain(AmazonS3 s3, String domainName) { @@ -182,14 +199,25 @@ public List getLocalDomainList() { lastModTime = System.currentTimeMillis(); } + // we are going to initialize our s3 client here since + // this is the first entry point before we start + // fetching all the domains individually + + awsS3Client = getS3Client(); + ArrayList domains = new ArrayList<>(); - listObjects(getS3Client(), domains, 0); + listObjects(awsS3Client, domains, 0); return domains; } @Override public Set getServerDomainList() { + // for the server domain list operation since it's called + // periodically by the thread to see if any domains have + // been deleted, we're going to get a new s3 client + // insetad of using our original client + HashSet domains = new HashSet<>(); listObjects(getS3Client(), domains, 0); return domains; @@ -212,6 +240,9 @@ public SignedDomains getUpdatedSignedDomains(StringBuilder lastModTimeBuffer) { // based on its last modification timestamp so we need to get // the full list and filter ourselves + // instead of using our fetched s3 client, we're going to + // obtain a new one to get the changes + AmazonS3 s3 = getS3Client(); ArrayList domains = new ArrayList<>(); listObjects(s3, domains, lastModTime); diff --git a/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/MockS3ChangeLogStore.java b/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/MockS3ChangeLogStore.java index d688b9e8247..aa847a1a910 100644 --- a/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/MockS3ChangeLogStore.java +++ b/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/MockS3ChangeLogStore.java @@ -22,17 +22,14 @@ import com.yahoo.athenz.zts.store.impl.S3ChangeLogStore; public class MockS3ChangeLogStore extends S3ChangeLogStore { - - AmazonS3 s3 = null; public MockS3ChangeLogStore(CloudStore cloudStore) { super(cloudStore); - s3 = mock(AmazonS3.class); + awsS3Client = mock(AmazonS3.class); } @Override AmazonS3 getS3Client() { - return s3; + return awsS3Client; } - } diff --git a/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStoreTest.java b/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStoreTest.java index a815226e810..6e12b75f54f 100644 --- a/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStoreTest.java +++ b/servers/zts/src/test/java/com/yahoo/athenz/zts/store/impl/S3ChangeLogStoreTest.java @@ -97,10 +97,10 @@ public void testListObjectsAllObjectsNoPage() { ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); ArrayList domains = new ArrayList<>(); - store.listObjects(store.s3, domains, 0); + store.listObjects(store.awsS3Client, domains, 0); assertEquals(domains.size(), 2); assertTrue(domains.contains("iaas")); @@ -125,10 +125,10 @@ public void testListObjectsAllObjectsNoPageModTime() { ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); ArrayList domains = new ArrayList<>(); - store.listObjects(store.s3, domains, (new Date(150)).getTime()); + store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime()); assertEquals(domains.size(), 1); assertTrue(domains.contains("iaas.athenz")); @@ -172,11 +172,11 @@ public void testListObjectsAllObjectsMultiplePages() { .thenReturn(true) .thenReturn(true) .thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); - when(store.s3.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing); ArrayList domains = new ArrayList<>(); - store.listObjects(store.s3, domains, 0); + store.listObjects(store.awsS3Client, domains, 0); assertEquals(domains.size(), 6); assertTrue(domains.contains("iaas")); @@ -231,11 +231,11 @@ public void testListObjectsAllObjectsMultiplePagesModTime() { .thenReturn(true) .thenReturn(true) .thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); - when(store.s3.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(objectListing); ArrayList domains = new ArrayList<>(); - store.listObjects(store.s3, domains, (new Date(150)).getTime()); + store.listObjects(store.awsS3Client, domains, (new Date(150)).getTime()); assertEquals(domains.size(), 3); assertTrue(domains.contains("cd.docker")); @@ -258,7 +258,7 @@ public void testGetLocalDomains() { ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); // verify that our last mod time is 0 before the call @@ -292,7 +292,7 @@ public void testGetServerDomains() { ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); // verify that our last mod time is 0 before the call @@ -314,25 +314,25 @@ public void testGetServerDomains() { @Test public void testGetSignedDomainNotFound() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(null); - when(store.s3.getObject(any(GetObjectRequest.class))).thenReturn((S3Object) null); + when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenReturn((S3Object) null); - assertNull(store.getSignedDomain(store.s3, "iaas")); + assertNull(store.getSignedDomain(store.awsS3Client, "iaas")); } @Test public void testGetSignedDomainClientException() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(null); - when(store.s3.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonClientException("failed client operation")); - assertNull(store.getSignedDomain(store.s3, "iaas")); + when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonClientException("failed client operation")); + assertNull(store.getSignedDomain(store.awsS3Client, "iaas")); } @Test public void testGetSignedDomainServiceException() { MockS3ChangeLogStore store = new MockS3ChangeLogStore(null); - when(store.s3.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonServiceException("failed server operation")); - assertNull(store.getSignedDomain(store.s3, "iaas")); + when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonServiceException("failed server operation")); + assertNull(store.getSignedDomain(store.awsS3Client, "iaas")); } private class MockS3ObjectInputStream extends S3ObjectInputStream { @@ -351,9 +351,9 @@ public void testGetSignedDomainInternal() throws IOException { S3Object object = mock(S3Object.class); when(object.getObjectContent()).thenReturn(s3Is); - when(store.s3.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object); + when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object); - SignedDomain signedDomain = store.getSignedDomain(store.s3, "iaas"); + SignedDomain signedDomain = store.getSignedDomain(store.awsS3Client, "iaas"); assertNotNull(signedDomain); DomainData domainData = signedDomain.getDomain(); assertNotNull(domainData); @@ -371,7 +371,29 @@ public void testGetSignedDomain() throws IOException { S3Object object = mock(S3Object.class); when(object.getObjectContent()).thenReturn(s3Is); - when(store.s3.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object); + when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object); + + SignedDomain signedDomain = store.getSignedDomain("iaas"); + assertNotNull(signedDomain); + DomainData domainData = signedDomain.getDomain(); + assertNotNull(domainData); + assertEquals(domainData.getName(), "iaas"); + is.close(); + } + + @Test + public void testGetSignedDomainException() throws IOException { + MockS3ChangeLogStore store = new MockS3ChangeLogStore(null); + + InputStream is = new FileInputStream("src/test/resources/iaas.json"); + MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null); + + S3Object object = mock(S3Object.class); + when(object.getObjectContent()).thenReturn(s3Is); + + // first call we return null, second call we return success + + when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenThrow(new AmazonServiceException("test")).thenReturn(object); SignedDomain signedDomain = store.getSignedDomain("iaas"); assertNotNull(signedDomain); @@ -399,7 +421,7 @@ public void testGetUpdatedSignedDomainsNoChanges() { ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); // set the last modification time to not return any of the domains store.lastModTime = (new Date(250)).getTime(); @@ -428,7 +450,7 @@ public void testGetUpdatedSignedDomainsWithChange() throws FileNotFoundException ObjectListing objectListing = mock(ObjectListing.class); when(objectListing.getObjectSummaries()).thenReturn(objectList); when(objectListing.isTruncated()).thenReturn(false); - when(store.s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); + when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing); InputStream is = new FileInputStream("src/test/resources/iaas.json"); MockS3ObjectInputStream s3Is = new MockS3ObjectInputStream(is, null); @@ -436,8 +458,8 @@ public void testGetUpdatedSignedDomainsWithChange() throws FileNotFoundException S3Object object = mock(S3Object.class); when(object.getObjectContent()).thenReturn(s3Is); - when(store.s3.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object); - when(store.s3.getObject("athenz-domain-sys.auth", "iaas.athenz")).thenReturn(object); + when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas")).thenReturn(object); + when(store.awsS3Client.getObject("athenz-domain-sys.auth", "iaas.athenz")).thenReturn(object); // set the last modification time to return one of the domains store.lastModTime = (new Date(150)).getTime();