Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade AWS SDK to v2 #107

Merged
merged 1 commit into from
Jul 8, 2024
Merged

Upgrade AWS SDK to v2 #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
<slf4j.version>2.0.7</slf4j.version>
<mockito.version>5.3.1</mockito.version>
<hamcrest.version>2.2</hamcrest.version>
<awssdk.version>1.12.475</awssdk.version>
<awssdk.version>2.26.15</awssdk.version>
<testcontainers.version>1.19.1</testcontainers.version>
</properties>

Expand All @@ -61,6 +61,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down Expand Up @@ -111,10 +118,14 @@
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${awssdk.version}</version>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kms</artifactId>
</dependency>


<!-- Google -->
<dependency>
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/zalando/baigan/repository/S3ConfigurationRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package org.zalando.baigan.repository;

import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.s3.AmazonS3;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.zalando.baigan.model.Configuration;
import org.zalando.baigan.repository.aws.S3FileLoader;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.s3.S3Client;

import javax.annotation.Nonnull;
import java.time.Duration;
Expand Down Expand Up @@ -35,7 +35,7 @@ public class S3ConfigurationRepository implements ConfigurationRepository {

S3ConfigurationRepository(@Nonnull final String bucketName, @Nonnull final String key,
final Duration refreshInterval, final ScheduledExecutorService executor,
final AmazonS3 s3Client, final AWSKMS kmsClient, ConfigurationParser configurationParser) {
final S3Client s3Client, final KmsClient kmsClient, ConfigurationParser configurationParser) {
checkNotNull(bucketName, "bucketName is required");
checkNotNull(key, "key is required");
checkArgument(!refreshInterval.isNegative(), "refreshInterval has to be >= 0");
Expand Down
26 changes: 12 additions & 14 deletions src/main/java/org/zalando/baigan/repository/S3ConfigurationRepositoryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package org.zalando.baigan.repository;

import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.AWSKMSClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.fasterxml.jackson.databind.ObjectMapper;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.s3.S3Client;

import javax.annotation.Nonnull;
import java.time.Duration;
Expand All @@ -25,8 +23,8 @@
public class S3ConfigurationRepositoryBuilder {

private ScheduledExecutorService executor;
private AmazonS3 s3Client;
private AWSKMS kmsClient;
private S3Client s3Client;
private KmsClient kmsClient;
private Duration refreshInterval = Duration.ofMinutes(1);
private String bucketName;
private String key;
Expand All @@ -39,20 +37,20 @@ public S3ConfigurationRepositoryBuilder(final ConfigurationParser configurationP

/**
* @param s3Client The S3 client to be used to fetch the configuration file.
* If the S3 client is not specified explicitly, the builder
* uses {@link AmazonS3ClientBuilder#defaultClient()}
* If the S3 client is not specified explicitly, Baigan builds a default
* client using {@link S3Client#builder()}.
*/
public S3ConfigurationRepositoryBuilder s3Client(final AmazonS3 s3Client) {
public S3ConfigurationRepositoryBuilder s3Client(final S3Client s3Client) {
this.s3Client = s3Client;
return this;
}

/**
* @param kmsClient The KMS client to be used to decrypt the configuration file.
* If the KMS client is not specified explicitly, the builder
* uses {@link AWSKMSClientBuilder#defaultClient()}
* If the KMS client is not specified explicitly, Baigan builds a default
* client using {@link KmsClient#builder()}.
*/
public S3ConfigurationRepositoryBuilder kmsClient(final AWSKMS kmsClient) {
public S3ConfigurationRepositoryBuilder kmsClient(final KmsClient kmsClient) {
this.kmsClient = kmsClient;
return this;
}
Expand Down Expand Up @@ -115,10 +113,10 @@ public S3ConfigurationRepository build() {
executor = new ScheduledThreadPoolExecutor(1);
}
if (s3Client == null) {
s3Client = AmazonS3ClientBuilder.defaultClient();
s3Client = S3Client.builder().build();
}
if (kmsClient == null) {
kmsClient = AWSKMSClientBuilder.defaultClient();
kmsClient = KmsClient.builder().build();
}
if (objectMapper != null) {
configurationParser.setObjectMapper(objectMapper);
Expand Down
32 changes: 20 additions & 12 deletions src/main/java/org/zalando/baigan/repository/aws/S3FileLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package org.zalando.baigan.repository.aws;

import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.model.DecryptRequest;
import com.amazonaws.services.kms.model.DependencyTimeoutException;
import com.amazonaws.services.kms.model.KMSInternalException;
import com.amazonaws.services.s3.AmazonS3;
import com.google.common.io.BaseEncoding;
import net.jodah.failsafe.Failsafe;
import net.jodah.failsafe.RetryPolicy;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.model.DecryptRequest;
import software.amazon.awssdk.services.kms.model.DependencyTimeoutException;
import software.amazon.awssdk.services.kms.model.KmsInternalException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;

import javax.annotation.Nonnull;
import java.nio.ByteBuffer;
Expand All @@ -28,25 +30,29 @@ public class S3FileLoader {
private static final int RETRY_SECONDS_WAIT = 10;

private final RetryPolicy<ByteBuffer> retryPolicy = new RetryPolicy<ByteBuffer>()
.handle(KMSInternalException.class)
.handle(KmsInternalException.class)
.handle(DependencyTimeoutException.class)
.withBackoff(1, RETRY_SECONDS_WAIT, SECONDS)
.withMaxRetries(MAX_RETRIES);

private final AmazonS3 s3Client;
private final AWSKMS kmsClient;
private final S3Client s3Client;
private final KmsClient kmsClient;
private final String bucketName;
private final String key;

public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull AmazonS3 s3Client, @Nonnull AWSKMS kmsClient) {
public S3FileLoader(@Nonnull String bucketName, @Nonnull String key, @Nonnull S3Client s3Client, @Nonnull KmsClient kmsClient) {
this.s3Client = s3Client;
this.kmsClient = kmsClient;
this.bucketName = bucketName;
this.key = key;
}

public String loadContent() {
final String configurationText = s3Client.getObjectAsString(bucketName, key);
final GetObjectRequest request = GetObjectRequest.builder()
.bucket(bucketName)
.key(key)
.build();
final String configurationText = s3Client.getObjectAsBytes(request).asUtf8String();
return decryptIfNecessary(configurationText);
}

Expand All @@ -68,8 +74,10 @@ private String decryptIfNecessary(final String candidate) {
}

private ByteBuffer decryptValue(final byte[] encryptedBytes) {
final DecryptRequest request = new DecryptRequest().withCiphertextBlob(ByteBuffer.wrap(encryptedBytes));
return Failsafe.with(retryPolicy).get(() -> kmsClient.decrypt(request).getPlaintext());
final DecryptRequest request = DecryptRequest.builder()
.ciphertextBlob(SdkBytes.fromByteArray(encryptedBytes))
.build();
return Failsafe.with(retryPolicy).get(() -> kmsClient.decrypt(request).plaintext().asByteBuffer());
}

private static Optional<byte[]> getEncryptedValue(final String value) {
Expand Down
99 changes: 51 additions & 48 deletions src/test/java/org/zalando/baigan/e2e/s3repo/S3ConfigurationRepositoryEnd2EndIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,5 @@
package org.zalando.baigan.e2e.s3repo;

import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.kms.AWSKMS;
import com.amazonaws.services.kms.AWSKMSClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
Expand All @@ -29,6 +19,15 @@
import org.zalando.baigan.e2e.configs.SomeConfiguration;
import org.zalando.baigan.repository.RepositoryFactory;
import org.zalando.baigan.repository.S3ConfigurationRepository;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;

import java.time.Duration;
import java.util.List;
Expand All @@ -48,11 +47,16 @@ public class S3ConfigurationRepositoryEnd2EndIT {
public static final String S3_CONFIG_BUCKET = "some-bucket";
public static final String S3_CONFIG_KEY = "some-key";

public static final PutObjectRequest PUT_OBJECT_REQUEST = PutObjectRequest.builder()
.bucket(S3_CONFIG_BUCKET)
.key(S3_CONFIG_KEY)
.build();

private static final Duration CONFIG_REFRESH_INTERVAL = Duration.ofMillis(100);
private static final long TIME_TO_WAIT_FOR_CONFIG_REFRESH = CONFIG_REFRESH_INTERVAL.plusMillis(100).toMillis();

@Autowired
private AmazonS3 s3;
private S3Client s3;

@Autowired
private SomeConfiguration someConfiguration;
Expand All @@ -69,9 +73,10 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio
assertThat(someConfiguration.topLevelGenerics(), nullValue());

s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
"[{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]"
PUT_OBJECT_REQUEST,
RequestBody.fromString(
"[{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]"
)
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.isThisTrue(), nullValue());
Expand All @@ -80,13 +85,14 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio
assertThat(someConfiguration.configList(), nullValue());

s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
PUT_OBJECT_REQUEST,
RequestBody.fromString(
"[{ \"alias\": \"some.configuration.some.config\", \"defaultValue\": {\"config_key\":\"a value\"}}," +
"{ \"alias\": \"some.non.existing.config\", \"defaultValue\": {\"other_config_key\":\"other value\"}}," +
"{ \"alias\": \"some.configuration.is.this.true\", \"defaultValue\": true}, " +
"{ \"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}, " +
"{ \"alias\": \"some.configuration.config.list\", \"defaultValue\": [\"A\",\"B\"]}]"
)
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.someConfig(), equalTo(new SomeConfigObject("a value")));
Expand All @@ -98,19 +104,19 @@ public void givenS3Configuration_whenConfigurationIsChangedOnS3_thenConfiguratio
@Test
public void givenS3Configuration_whenTheS3FileIsUpdatedWithInvalidConfig_thenTheConfigurationIsNotUpdated() throws InterruptedException {
s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
PUT_OBJECT_REQUEST,
RequestBody.fromString(
"[{\"alias\": \"some.configuration.is.this.true\", \"defaultValue\": true}, " +
"{\"alias\": \"some.configuration.some.value\", \"defaultValue\": \"some value\"}]"
)
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.isThisTrue(), equalTo(true));
assertThat(someConfiguration.someValue(), equalTo("some value"));

s3.putObject(
S3_CONFIG_BUCKET,
S3_CONFIG_KEY,
"an: invalid\"} config"
PUT_OBJECT_REQUEST,
RequestBody.fromString("an: invalid\"} config")
);
Thread.sleep(TIME_TO_WAIT_FOR_CONFIG_REFRESH);
assertThat(someConfiguration.isThisTrue(), equalTo(true));
Expand All @@ -135,11 +141,11 @@ ScheduledThreadPoolExecutor baiganRefresherPoolExecutor() {
@Bean
S3ConfigurationRepository configurationRepository(
RepositoryFactory repositoryFactory,
AmazonS3 amazonS3,
AWSKMS kms,
S3Client amazonS3,
KmsClient kms,
ScheduledThreadPoolExecutor executorService
) {
amazonS3.putObject(S3_CONFIG_BUCKET, S3_CONFIG_KEY, "[]");
amazonS3.putObject(PUT_OBJECT_REQUEST, RequestBody.fromString("[]"));
return repositoryFactory.s3ConfigurationRepository()
.bucketName(S3_CONFIG_BUCKET)
.key(S3_CONFIG_KEY)
Expand All @@ -153,38 +159,35 @@ S3ConfigurationRepository configurationRepository(
@Container
private static final LocalStackContainer localstack = new LocalStackContainer(
DockerImageName.parse("localstack/localstack:2.1.0")
).withServices(S3, KMS).withEnv("DEFAULT_REGION", Regions.EU_CENTRAL_1.getName());
).withServices(S3, KMS).withEnv("DEFAULT_REGION", Region.EU_CENTRAL_1.id());

@Bean
AWSKMS kms() {
KmsClient kms() {
localstack.start();
return AWSKMSClientBuilder.standard().withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(
localstack.getEndpointOverride(KMS).toString(), localstack.getRegion()
)
).build();
return KmsClient.builder()
.endpointOverride(localstack.getEndpoint())
.region(Region.of(localstack.getRegion()))
.build();
}

@Bean
AmazonS3 amazonS3() {
S3Client amazonS3() {
localstack.start();
AmazonS3 s3 = AmazonS3ClientBuilder.standard().withEndpointConfiguration(
new AwsClientBuilder.EndpointConfiguration(
localstack.getEndpointOverride(S3).toString(), localstack.getRegion()
)
).withCredentials(
new AWSStaticCredentialsProvider(
new BasicAWSCredentials(localstack.getAccessKey(), localstack.getSecretKey())
S3Client s3 = S3Client
.builder()
.endpointOverride(localstack.getEndpoint())
.credentialsProvider(
StaticCredentialsProvider.create(
AwsBasicCredentials.create(localstack.getAccessKey(), localstack.getSecretKey())
)
)
).build();

try {
s3.createBucket(new CreateBucketRequest(S3_CONFIG_BUCKET, localstack.getRegion()));
} catch (AmazonS3Exception e) {
if (!e.getErrorCode().equals("BucketAlreadyOwnedByYou")) {
throw e;
}
}
.region(Region.of(localstack.getRegion()))
.build();

CreateBucketRequest request = CreateBucketRequest.builder()
.bucket(S3_CONFIG_BUCKET)
.build();
s3.createBucket(request);

return s3;
}
Expand Down
Loading