Skip to content

Commit

Permalink
Upgrade AWS SDK to v2
Browse files Browse the repository at this point in the history 
Initial attempt
  • Loading branch information
@yuppie-flu
yuppie-flu authored and Kirill Kozlov committed Jul 5, 2024
1 parent 4c2a278 commit feba83a
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 81 deletions.
19 changes: 15 additions & 4 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
<slf4j.version>2.0.7</slf4j.version>
<mockito.version>5.3.1</mockito.version>
<hamcrest.version>2.2</hamcrest.version>
<awssdk.version>1.12.475</awssdk.version>
<awssdk.version>2.26.15</awssdk.version>
<testcontainers.version>1.19.1</testcontainers.version>
</properties>

Expand All @@ -61,6 +61,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down Expand Up @@ -111,10 +118,14 @@
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${awssdk.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kms</artifactId>
</dependency>


<!-- Google -->
<dependency>
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/zalando/baigan/repository/S3ConfigurationRepository.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package org.zalando.baigan.repository;

import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.s3.AmazonS3;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.zalando.baigan.model.Configuration;
import org.zalando.baigan.repository.aws.S3FileLoader;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.s3.S3Client;

import javax.annotation.Nonnull;
import java.time.Duration;
Expand Down Expand Up @@ -35,7 +35,7 @@ public class S3ConfigurationRepository implements ConfigurationRepository {

S3ConfigurationRepository(@Nonnull final String bucketName, @Nonnull final String key,
final Duration refreshInterval, final ScheduledExecutorService executor,
final AmazonS3 s3Client, final AWSKMS kmsClient, ConfigurationParser configurationParser) {
final S3Client s3Client, final KmsClient kmsClient, ConfigurationParser configurationParser) {
checkNotNull(bucketName, "bucketName is required");
checkNotNull(key, "key is required");
checkArgument(!refreshInterval.isNegative(), "refreshInterval has to be >= 0");
Expand Down
26 changes: 12 additions & 14 deletions src/main/java/org/zalando/baigan/repository/S3ConfigurationRepositoryBuilder.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package org.zalando.baigan.repository;

import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.AWSKMSClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.s3.S3Client;

import javax.annotation.Nonnull;
import java.time.Duration;
Expand All @@ -25,8 +23,8 @@
public class S3ConfigurationRepositoryBuilder {

private ScheduledExecutorService executor;
private AmazonS3 s3Client;
private AWSKMS kmsClient;
private S3Client s3Client;
private KmsClient kmsClient;
private Duration refreshInterval = Duration.ofMinutes(1);
private String bucketName;
private String key;
Expand All @@ -39,20 +37,20 @@ public S3ConfigurationRepositoryBuilder(final ConfigurationParser configurationP

/**
* @param s3Client The S3 client to be used to fetch the configuration file.
* If the S3 client is not specified explicitly, the builder
* uses {@link AmazonS3ClientBuilder#defaultClient()}
* If the S3 client is not specified explicitly, Baigan builds a default
* client using {@link S3Client#builder()}.
*/
public S3ConfigurationRepositoryBuilder s3Client(final AmazonS3 s3Client) {
public S3ConfigurationRepositoryBuilder s3Client(final S3Client s3Client) {
this.s3Client = s3Client;
return this;
}

/**
* @param kmsClient The KMS client to be used to decrypt the configuration file.
* If the KMS client is not specified explicitly, the builder
* uses {@link AWSKMSClientBuilder#defaultClient()}
* If the KMS client is not specified explicitly, Baigan builds a default
* client using {@link KmsClient#builder()}.
*/
public S3ConfigurationRepositoryBuilder kmsClient(final AWSKMS kmsClient) {
public S3ConfigurationRepositoryBuilder kmsClient(final KmsClient kmsClient) {
this.kmsClient = kmsClient;
return this;
}
Expand Down Expand Up @@ -115,10 +113,10 @@ public S3ConfigurationRepository build() {
executor = new ScheduledThreadPoolExecutor(1);
}
if (s3Client == null) {
s3Client = AmazonS3ClientBuilder.defaultClient();
s3Client = S3Client.builder().build();
}
if (kmsClient == null) {
kmsClient = AWSKMSClientBuilder.defaultClient();
kmsClient = KmsClient.builder().build();
}
if (objectMapper != null) {
configurationParser.setObjectMapper(objectMapper);
Expand Down
32 changes: 20 additions & 12 deletions src/main/java/org/zalando/baigan/repository/aws/S3FileLoader.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package org.zalando.baigan.repository.aws;

import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.model.DecryptRequest;
import com.amazonaws.services.kms.model.DependencyTimeoutException;
import com.amazonaws.services.kms.model.KMSInternalException;
import com.amazonaws.services.s3.AmazonS3;
import com.google.common.io.BaseEncoding;
import net.jodah.failsafe.Failsafe;
import net.jodah.failsafe.RetryPolicy;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.model.DecryptRequest;
import software.amazon.awssdk.services.kms.model.DependencyTimeoutException;
import software.amazon.awssdk.services.kms.model.KmsInternalException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;

import javax.annotation.Nonnull;
import java.nio.ByteBuffer;
Expand All @@ -28,25 +30,29 @@ public class S3FileLoader {
private static final int RETRY_SECONDS_WAIT = 10;

private final RetryPolicy<ByteBuffer> retryPolicy = new RetryPolicy<ByteBuffer>()
.handle(KMSInternalException.class)
.handle(KmsInternalException.class)
.handle(DependencyTimeoutException.class)
.withBackoff(1, RETRY_SECONDS_WAIT, SECONDS)
.withMaxRetries(MAX_RETRIES);

private final AmazonS3 s3Client;
private final AWSKMS kmsClient;
private final S3Client s3Client;
private final KmsClient kmsClient;
private final String bucketName;
private final String key;

public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull AmazonS3 s3Client, @Nonnull AWSKMS kmsClient) {
public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull S3Client s3Client, @Nonnull KmsClient kmsClient) {
this.s3Client = s3Client;
this.kmsClient = kmsClient;
this.bucketName = bucketName;
this.key = key;
}

public String loadContent() {
final String configurationText = s3Client.getObjectAsString(bucketName, key);
final GetObjectRequest request = GetObjectRequest.builder()
.bucket(bucketName)
.key(key)
.build();
final String configurationText = s3Client.getObjectAsBytes(request).asUtf8String();
return decryptIfNecessary(configurationText);
}

Expand All @@ -68,8 +74,10 @@ private String decryptIfNecessary(final String candidate) {
}

private ByteBuffer decryptValue(final byte[] encryptedBytes) {
final DecryptRequest request = new DecryptRequest().withCiphertextBlob(ByteBuffer.wrap(encryptedBytes));
return Failsafe.with(retryPolicy).get(() -> kmsClient.decrypt(request).getPlaintext());
final DecryptRequest request = DecryptRequest.builder()
.ciphertextBlob(SdkBytes.fromByteArray(encryptedBytes))
.build();
return Failsafe.with(retryPolicy).get(() -> kmsClient.decrypt(request).plaintext().asByteBuffer());
}

private static Optional<byte[]> getEncryptedValue(final String value) {
Expand Down
99 changes: 51 additions & 48 deletions src/test/java/org/zalando/baigan/e2e/s3repo/S3ConfigurationRepositoryEnd2EndIT.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,5 @@
package org.zalando.baigan.e2e.s3repo;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.AWSKMSClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
Expand All @@ -29,6 +19,15 @@
import org.zalando.baigan.e2e.configs.SomeConfiguration;
import org.zalando.baigan.repository.RepositoryFactory;
import org.zalando.baigan.repository.S3ConfigurationRepository;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;

import java.time.Duration;
import java.util.List;
Expand All @@ -48,11 +47,16 @@ public class S3ConfigurationRepositoryEnd2EndIT {
public static final String S3_CONFIG_BUCKET = "some-bucket";
public static final String S3_CONFIG_KEY = "some-key";

public static final PutObjectRequest PUT_OBJECT_REQUEST = PutObjectRequest.builder()
.bucket(S3_CONFIG_BUCKET)
.key(S3_CONFIG_KEY)
.build();

private static final Duration CONFIG_REFRESH_INTERVAL = Duration.ofMillis(100);
private static final long TIME_TO_WAIT_FOR_CONFIG_REFRESH = CONFIG_REFRESH_INTERVAL.plusMillis(100).toMillis();

@Autowired
private AmazonS3 s3;
private S3Client s3;

@Autowired
private SomeConfiguration someConfiguration;
Expand All @@ -69,9 +73,10 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio
assertThat(someConfiguration.topLevelGenerics(), nullValue());

s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
"[{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]"
PUT_OBJECT_REQUEST,
RequestBody.fromString(
"[{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]"
)
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.isThisTrue(), nullValue());
Expand All @@ -80,13 +85,14 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio
assertThat(someConfiguration.configList(), nullValue());

s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
PUT_OBJECT_REQUEST,
RequestBody.fromString(
"[{ \"alias\": \"some.configuration.some.config\", \"defaultValue\": {\"config_key\":\"a value\"}}," +
"{ \"alias\": \"some.non.existing.config\", \"defaultValue\": {\"other_config_key\":\"other value\"}}," +
"{ \"alias\": \"some.configuration.is.this.true\", \"defaultValue\": true}, " +
"{ \"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}, " +
"{ \"alias\": \"some.configuration.config.list\", \"defaultValue\": [\"A\",\"B\"]}]"
)
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.someConfig(), equalTo(new SomeConfigObject("a value")));
Expand All @@ -98,19 +104,19 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio
@Test
public void givenS3Configuration_whenTheS3FileIsUpdatedWithInvalidConfig_thenTheConfigurationIsNotUpdated() throws InterruptedException {
s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
PUT_OBJECT_REQUEST,
RequestBody.fromString(
"[{\"alias\": \"some.configuration.is.this.true\", \"defaultValue\": true}, " +
"{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]"
)
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.isThisTrue(), equalTo(true));
assertThat(someConfiguration.someValue(), equalTo("some value"));

s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
"an: invalid\"} config"
PUT_OBJECT_REQUEST,
RequestBody.fromString("an: invalid\"} config")
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.isThisTrue(), equalTo(true));
Expand All @@ -135,11 +141,11 @@ ScheduledThreadPoolExecutor baiganRefresherPoolExecutor() {
@Bean
S3ConfigurationRepository configurationRepository(
RepositoryFactory repositoryFactory,
AmazonS3 amazonS3,
AWSKMS kms,
S3Client amazonS3,
KmsClient kms,
ScheduledThreadPoolExecutor executorService
) {
amazonS3.putObject(S3_CONFIG_BUCKET, S3_CONFIG_KEY, "[]");
amazonS3.putObject(PUT_OBJECT_REQUEST, RequestBody.fromString("[]"));
return repositoryFactory.s3ConfigurationRepository()
.bucketName(S3_CONFIG_BUCKET)
.key(S3_CONFIG_KEY)
Expand All @@ -153,38 +159,35 @@ S3ConfigurationRepository configurationRepository(
@Container
private static final LocalStackContainer localstack = new LocalStackContainer(
DockerImageName.parse("localstack/localstack:2.1.0")
).withServices(S3, KMS).withEnv("DEFAULT_REGION", Regions.EU_CENTRAL_1.getName());
).withServices(S3, KMS).withEnv("DEFAULT_REGION", Region.EU_CENTRAL_1.id());

@Bean
AWSKMS kms() {
KmsClient kms() {
localstack.start();
return AWSKMSClientBuilder.standard().withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(
localstack.getEndpointOverride(KMS).toString(), localstack.getRegion()
)
).build();
return KmsClient.builder()
.endpointOverride(localstack.getEndpoint())
.region(Region.of(localstack.getRegion()))
.build();
}

@Bean
AmazonS3 amazonS3() {
S3Client amazonS3() {
localstack.start();
AmazonS3 s3 = AmazonS3ClientBuilder.standard().withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(
localstack.getEndpointOverride(S3).toString(), localstack.getRegion()
)
).withCredentials(
new AWSStaticCredentialsProvider(
new BasicAWSCredentials(localstack.getAccessKey(), localstack.getSecretKey())
S3Client s3 = S3Client
.builder()
.endpointOverride(localstack.getEndpoint())
.credentialsProvider(
StaticCredentialsProvider.create(
AwsBasicCredentials.create(localstack.getAccessKey(), localstack.getSecretKey())
)
)
).build();

try {
s3.createBucket(new CreateBucketRequest(S3_CONFIG_BUCKET, localstack.getRegion()));
} catch (AmazonS3Exception e) {
if (!e.getErrorCode().equals("BucketAlreadyOwnedByYou")) {
throw e;
}
}
.region(Region.of(localstack.getRegion()))
.build();

CreateBucketRequest request = CreateBucketRequest.builder()
.bucket(S3_CONFIG_BUCKET)
.build();
s3.createBucket(request);

return s3;
}
Expand Down

0 comments on commit feba83a

Please sign in to comment.