From d8598db2065f9ebbb89f0452fea349fedec37840 Mon Sep 17 00:00:00 2001 From: Colin Redmond Date: Tue, 18 Jul 2023 22:58:01 -0700 Subject: [PATCH] feat: map decoder supports flattening maps and arrays. https://github.com/gestalt-config/gestalt/issues/99 fix: remove un-needed check if key is in cache when we check if it is not null. chore: Use test containers to replace aws s3 mock. --- .editorconfig | 4 +- build.gradle.kts | 2 +- buildSrc/build.gradle.kts | 2 +- ...kotlin-code-quality-conventions.gradle.kts | 6 +- config/detekt/config.yml | 2 +- gestalt-aws/build.gradle.kts | 22 +-- .../aws/s3/S3ConfigSourceDockerTest.java | 101 +++++++++++ ...eTest.java => S3ConfigSourceMockTest.java} | 57 +----- .../github/gestalt/config/GestaltCache.java | 2 +- .../github/gestalt/config/GestaltCore.java | 4 +- .../gestalt/config/decoder/MapDecoder.java | 95 ++++++---- .../github/gestalt/config/node/ArrayNode.java | 10 ++ .../gestalt/config/utils/ClassUtils.java | 29 +++ .../config/decoder/MapDecoderTest.java | 167 ++++++++++++++++-- .../build.gradle.kts | 29 ++- .../gestalt-sample/build.gradle.kts | 73 ++++---- .../config/integration/GestaltSample.java | 70 +++++++- gestalt-google-cloud/build.gradle.kts | 21 +-- gradle/libs.versions.toml | 32 ++-- 19 files changed, 528 insertions(+), 200 deletions(-) create mode 100644 gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceDockerTest.java rename gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/{S3ConfigSourceTest.java => S3ConfigSourceMockTest.java} (62%) diff --git a/.editorconfig b/.editorconfig index 51e3ca3d6..73c7dc0e2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -20,12 +20,12 @@ max_line_length = 140 ij_visual_guides = 140 ij_continuation_indent_size = 4 -[*.{kt, kts}] +[*.{kt,kts}] indent_size = 4 max_line_length = 140 ij_visual_guides = 140 ij_continuation_indent_size = 4 # Markdown & YAML Settings -[*.{md, yml, yaml}] +[*.{md,yml,yaml}] trim_trailing_whitespace = false diff --git a/build.gradle.kts b/build.gradle.kts index a330ed9bd..03f037420 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { allprojects { group = "com.github.gestalt-config" - version = "0.22.0" + version = "0.22.1" } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index ef833bde9..025a48d6b 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -9,7 +9,7 @@ plugins { } kotlin { - jvmToolchain(libs.versions.java.get().toInt()) + jvmToolchain(libs.versions.java.get().toInt()) } dependencies { diff --git a/buildSrc/src/main/kotlin/gestalt.kotlin-code-quality-conventions.gradle.kts b/buildSrc/src/main/kotlin/gestalt.kotlin-code-quality-conventions.gradle.kts index 6f1c64f1c..5439d7c67 100644 --- a/buildSrc/src/main/kotlin/gestalt.kotlin-code-quality-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/gestalt.kotlin-code-quality-conventions.gradle.kts @@ -14,7 +14,7 @@ dependencies { detekt { toolVersion = libs.versions.detekt.get() - config = files(project.rootDir.resolve("config/detekt/config.yml")) + config.setFrom("$rootDir/config/detekt/config.yml") debug = false parallel = false } @@ -25,7 +25,7 @@ tasks.withType { exclude(".*/build/.*") reports { - xml.required.set(false) - html.required.set(false) + xml.required.set(true) + html.required.set(true) } } diff --git a/config/detekt/config.yml b/config/detekt/config.yml index 56ecaddea..2e0a3e860 100644 --- a/config/detekt/config.yml +++ b/config/detekt/config.yml @@ -731,7 +731,7 @@ style: UntilInsteadOfRangeTo: active: false UnusedImports: - active: false + active: true UnusedParameter: active: true allowedNames: 'ignored|expected' diff --git a/gestalt-aws/build.gradle.kts b/gestalt-aws/build.gradle.kts index 791a293b2..1151d5308 100644 --- a/gestalt-aws/build.gradle.kts +++ b/gestalt-aws/build.gradle.kts @@ -1,17 +1,19 @@ plugins { - id("gestalt.java-library-conventions") - id("gestalt.java-test-conventions") - id("gestalt.java-code-quality-conventions") - id("gestalt.java-publish-conventions") + id("gestalt.java-library-conventions") + id("gestalt.java-test-conventions") + id("gestalt.java-code-quality-conventions") + id("gestalt.java-publish-conventions") } dependencies { - implementation(project(":gestalt-core")) - implementation(libs.aws.s3) - implementation(libs.aws.secret) - implementation(libs.aws.url.client) - implementation(libs.jackson.databind) - testImplementation(libs.aws.mock) + implementation(project(":gestalt-core")) + implementation(platform(libs.aws.bom)) + api(libs.aws.s3) + api(libs.aws.secret) + api(libs.aws.url.client) + implementation(libs.jackson.databind) + testImplementation(libs.aws.mock) + testImplementation(libs.testcontainers.junit5) } diff --git a/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceDockerTest.java b/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceDockerTest.java new file mode 100644 index 000000000..e6712158d --- /dev/null +++ b/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceDockerTest.java @@ -0,0 +1,101 @@ +package org.github.gestalt.config.aws.s3; + +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; +import org.github.gestalt.config.exceptions.GestaltException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.AttributeMap; + +import java.io.File; +import java.net.URI; +import java.util.Collection; + +import static java.util.Arrays.asList; +import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; + +@Testcontainers +class S3ConfigSourceDockerTest { + private static final String BUCKET_NAME = "testbucket"; + private static final String BUCKET_NAME_2 = "testbucket2"; + + private static final String S3MOCK_VERSION = System.getProperty("s3mock.version", "latest"); + private static final Collection INITIAL_BUCKET_NAMES = asList(BUCKET_NAME, BUCKET_NAME_2); + private static final String TEST_ENC_KEYREF = + "arn:aws:kms:us-east-1:1234567890:key/valid-test-key-ref"; + + private static final String UPLOAD_FILE_NAME = "src/test/resources/default.properties"; + + private S3Client s3Client; + + @Container + private static final S3MockContainer s3Mock = + new S3MockContainer(S3MOCK_VERSION) + .withValidKmsKeys(TEST_ENC_KEYREF) + .withInitialBuckets(String.join(",", INITIAL_BUCKET_NAMES)); + + @BeforeEach + void setUp() { + // Must create S3Client after S3MockContainer is started, otherwise we can't request the random + // locally mapped port for the endpoint + var endpoint = s3Mock.getHttpsEndpoint(); + s3Client = createS3ClientV2(endpoint); + } + + protected S3Client createS3ClientV2(String endpoint) { + return S3Client.builder() + .region(Region.of("us-east-1")) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("foo", "bar"))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .endpointOverride(URI.create(endpoint)) + .httpClient(UrlConnectionHttpClient.builder().buildWithDefaults( + AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, Boolean.TRUE).build())) + .build(); + } + + @Test + void loadFile() throws GestaltException { + + final File uploadFile = new File(UPLOAD_FILE_NAME); + + s3Client.putObject( + PutObjectRequest.builder().bucket(BUCKET_NAME).key(uploadFile.getName()).build(), + RequestBody.fromFile(uploadFile)); + + + S3ConfigSource source = new S3ConfigSource(s3Client, BUCKET_NAME, uploadFile.getName()); + + Assertions.assertTrue(source.hasStream()); + Assertions.assertNotNull(source.loadStream()); + } + + @Test + void loadFileDoesNotExist() throws GestaltException { + + final File uploadFile = new File(UPLOAD_FILE_NAME); + + s3Client.putObject( + PutObjectRequest.builder().bucket(BUCKET_NAME_2).key(uploadFile.getName()).build(), + RequestBody.fromFile(uploadFile)); + + + S3ConfigSource source = new S3ConfigSource(s3Client, BUCKET_NAME_2, uploadFile.getName() + ".noMatch"); + + Assertions.assertTrue(source.hasStream()); + GestaltException ex = Assertions.assertThrows(GestaltException.class, source::loadStream); + + Assertions.assertEquals("Exception loading S3 key: default.properties.noMatch, bucket: testbucket2, " + + "with error: The specified key does not exist.", ex.getMessage()); + } +} diff --git a/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceTest.java b/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceMockTest.java similarity index 62% rename from gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceTest.java rename to gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceMockTest.java index 27a1aacb4..ca06730d8 100644 --- a/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceTest.java +++ b/gestalt-aws/src/test/java/org/github/gestalt/config/aws/s3/S3ConfigSourceMockTest.java @@ -1,65 +1,26 @@ package org.github.gestalt.config.aws.s3; -import com.adobe.testing.s3mock.junit5.S3MockExtension; import org.github.gestalt.config.exceptions.GestaltException; import org.github.gestalt.config.tag.Tags; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import software.amazon.awssdk.core.sync.RequestBody; +import org.mockito.Mockito; +import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import java.io.File; - -class S3ConfigSourceTest { - - @RegisterExtension - static final S3MockExtension S3_MOCK = - S3MockExtension.builder().silent().withSecureConnection(false).build(); +@Testcontainers +class S3ConfigSourceMockTest { private static final String BUCKET_NAME = "testbucket"; - private static final String BUCKET_NAME_2 = "testbucket2"; - private static final String UPLOAD_FILE_NAME = "src/test/resources/default.properties"; - - private final S3Client s3Client = S3_MOCK.createS3ClientV2(); - - @Test - void loadFile() throws GestaltException { - - final File uploadFile = new File(UPLOAD_FILE_NAME); - - s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); - s3Client.putObject( - PutObjectRequest.builder().bucket(BUCKET_NAME).key(uploadFile.getName()).build(), - RequestBody.fromFile(uploadFile)); - - - S3ConfigSource source = new S3ConfigSource(s3Client, BUCKET_NAME, uploadFile.getName()); - - Assertions.assertTrue(source.hasStream()); - Assertions.assertNotNull(source.loadStream()); - } - - @Test - void loadFileDoesNotExist() throws GestaltException { - - final File uploadFile = new File(UPLOAD_FILE_NAME); - - s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME_2).build()); - s3Client.putObject( - PutObjectRequest.builder().bucket(BUCKET_NAME_2).key(uploadFile.getName()).build(), - RequestBody.fromFile(uploadFile)); + private static final String UPLOAD_FILE_NAME = "src/test/resources/default.properties"; - S3ConfigSource source = new S3ConfigSource(s3Client, BUCKET_NAME_2, uploadFile.getName() + ".noMatch"); - Assertions.assertTrue(source.hasStream()); - GestaltException ex = Assertions.assertThrows(GestaltException.class, source::loadStream); + private final S3Client s3Client = Mockito.mock(); - Assertions.assertEquals("Exception loading S3 key: default.properties.noMatch, bucket: testbucket2, " + - "with error: The specified key does not exist.", ex.getMessage()); + @BeforeEach + void setUp() { } @Test diff --git a/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCache.java b/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCache.java index e8ef232ee..059ae2349 100644 --- a/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCache.java +++ b/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCache.java @@ -57,7 +57,7 @@ public T getConfig(String path, TypeCapture klass) throws GestaltExceptio @SuppressWarnings("unchecked") public T getConfig(String path, TypeCapture klass, Tags tags) throws GestaltException { Triple, Tags> key = new Triple<>(path, klass, tags); - if (cache.containsKey(key) && cache.get(key) != null) { + if (cache.get(key) != null) { return (T) cache.get(key); } else { T result = delegate.getConfig(path, klass, tags); diff --git a/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCore.java b/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCore.java index 7334f4f80..bdbbde46a 100644 --- a/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCore.java +++ b/gestalt-core/src/main/java/org/github/gestalt/config/GestaltCore.java @@ -334,7 +334,7 @@ private T getConfigInternal(String path, boolean failOnErrors, T defaultVal, if (logger.isLoggable(gestaltConfig.getLogLevelForMissingValuesWhenDefaultOrOptional())) { String errorMsg = ErrorsUtil.buildErrorMessage("Failed getting config path: " + combinedPath + ", for class: " + klass.getName() + " returning empty Optional", results.getErrors()); - logger.log(DEBUG, errorMsg); + logger.log(gestaltConfig.getLogLevelForMissingValuesWhenDefaultOrOptional(), errorMsg); } return defaultVal; @@ -354,7 +354,7 @@ private T getConfigInternal(String path, boolean failOnErrors, T defaultVal, if (logger.isLoggable(gestaltConfig.getLogLevelForMissingValuesWhenDefaultOrOptional())) { String errorMsg = ErrorsUtil.buildErrorMessage("No results for Optional config path: " + combinedPath + ", and class: " + klass.getName() + " returning empty Optional", tokens.getErrors()); - logger.log(DEBUG, errorMsg); + logger.log(gestaltConfig.getLogLevelForMissingValuesWhenDefaultOrOptional(), errorMsg); } if (failOnErrors) { diff --git a/gestalt-core/src/main/java/org/github/gestalt/config/decoder/MapDecoder.java b/gestalt-core/src/main/java/org/github/gestalt/config/decoder/MapDecoder.java index c7c233a85..a9630fe3e 100644 --- a/gestalt-core/src/main/java/org/github/gestalt/config/decoder/MapDecoder.java +++ b/gestalt-core/src/main/java/org/github/gestalt/config/decoder/MapDecoder.java @@ -1,15 +1,18 @@ package org.github.gestalt.config.decoder; import org.github.gestalt.config.entity.ValidationError; +import org.github.gestalt.config.node.ArrayNode; import org.github.gestalt.config.node.ConfigNode; import org.github.gestalt.config.node.LeafNode; import org.github.gestalt.config.node.MapNode; import org.github.gestalt.config.reflect.TypeCapture; +import org.github.gestalt.config.utils.ClassUtils; import org.github.gestalt.config.utils.Pair; import org.github.gestalt.config.utils.PathUtil; import org.github.gestalt.config.utils.ValidateOf; import java.util.*; +import java.util.stream.Stream; /** * Decode a Map. Assumes that the key is a simple class that can be decoded from a single string. ie a Boolean, String, Int. @@ -49,37 +52,42 @@ public boolean matches(TypeCapture type) { List errors = new ArrayList<>(); - Map map = mapNode.getMapNode().entrySet().stream() - .map(it -> { - String key = it.getKey(); - if (key == null) { - errors.add(new ValidationError.DecodersMapKeyNull(path)); - return null; - } - - String nextPath = PathUtil.pathForKey(path, key); - ValidateOf keyValidate = decoderService.decodeNode(nextPath, new LeafNode(key), - (TypeCapture) keyType); - ValidateOf valueValidate = decoderService.decodeNode(nextPath, it.getValue(), - (TypeCapture) valueType); - - errors.addAll(keyValidate.getErrors()); - errors.addAll(valueValidate.getErrors()); - - if (!keyValidate.hasResults()) { - errors.add(new ValidationError.DecodersMapKeyNull(nextPath)); - } - if (!valueValidate.hasResults()) { - errors.add(new ValidationError.DecodersMapValueNull(nextPath)); - } - - if (keyValidate.hasResults()) { - return new Pair<>(keyValidate.results(), valueValidate.results()); - } - return null; - }) - .filter(Objects::nonNull) - .collect(HashMap::new, (m, v) -> m.put(v.getFirst(), v.getSecond()), HashMap::putAll); + var stream = mapNode.getMapNode().entrySet().stream(); + + // if the value of the map is a primitive or a wrapper, flat map any entries that are map nodes. + // if the value is a class, then we want to decode the map nodes into an object + if (ClassUtils.isPrimitiveOrWrapper(valueType.getRawType())) { + stream = stream.flatMap(it -> convertMapToStream(it.getKey(), it)); + } + + Map map = stream.map(it -> { + String key = it.getKey(); + if (key == null) { + errors.add(new ValidationError.DecodersMapKeyNull(path)); + return null; + } + + String nextPath = PathUtil.pathForKey(path, key); + ValidateOf keyValidate = decoderService.decodeNode(nextPath, new LeafNode(key), (TypeCapture) keyType); + ValidateOf valueValidate = decoderService.decodeNode(nextPath, it.getValue(), (TypeCapture) valueType); + + errors.addAll(keyValidate.getErrors()); + errors.addAll(valueValidate.getErrors()); + + if (!keyValidate.hasResults()) { + errors.add(new ValidationError.DecodersMapKeyNull(nextPath)); + } + if (!valueValidate.hasResults()) { + errors.add(new ValidationError.DecodersMapValueNull(nextPath)); + } + + if (keyValidate.hasResults()) { + return new Pair<>(keyValidate.results(), valueValidate.results()); + } + return null; + }) + .filter(Objects::nonNull) + .collect(HashMap::new, (m, v) -> m.put(v.getFirst(), v.getSecond()), HashMap::putAll); return ValidateOf.validateOf(map, errors); @@ -90,4 +98,29 @@ public boolean matches(TypeCapture type) { return results; } + private Stream> convertMapToStream(String path, Map.Entry entry) { + // if the key or entry is null, return the current entry and let later code deal with the null value. + if (path == null || entry.getValue() == null) { + return Stream.of(entry); + } else if (entry.getValue() instanceof MapNode) { + MapNode node = (MapNode) entry.getValue(); + + return node.getMapNode().entrySet().stream().flatMap(it -> convertMapToStream(path + "." + it.getKey(), it)); + } else if (entry.getValue() instanceof ArrayNode) { + ArrayNode node = (ArrayNode) entry.getValue(); + + Stream> stream = Stream.of(); + List nodes = node.getArray(); + + for (int i = 0; i < nodes.size(); i++) { + stream = Stream.concat(stream, convertMapToStream(path + "[" + i + "]", Map.entry("[" + i + "]", nodes.get(i)))); + } + + return stream; + } else if (entry.getValue() instanceof LeafNode) { + return Stream.of(Map.entry(path, entry.getValue())); + } else { + return Stream.of(); + } + } } diff --git a/gestalt-core/src/main/java/org/github/gestalt/config/node/ArrayNode.java b/gestalt-core/src/main/java/org/github/gestalt/config/node/ArrayNode.java index b07af468c..33e13957a 100644 --- a/gestalt-core/src/main/java/org/github/gestalt/config/node/ArrayNode.java +++ b/gestalt-core/src/main/java/org/github/gestalt/config/node/ArrayNode.java @@ -11,6 +11,7 @@ * @author Colin Redmond (c) 2023. */ public class ArrayNode implements ConfigNode { + private final List values; /** @@ -41,6 +42,15 @@ public Optional getIndex(int index) { } } + /** + * get the underlying array for the node. + * + * @return the underlying array + */ + public List getArray() { + return values; + } + @Override public Optional getKey(String key) { return Optional.empty(); diff --git a/gestalt-core/src/main/java/org/github/gestalt/config/utils/ClassUtils.java b/gestalt-core/src/main/java/org/github/gestalt/config/utils/ClassUtils.java index f89b0c2a1..95064b222 100644 --- a/gestalt-core/src/main/java/org/github/gestalt/config/utils/ClassUtils.java +++ b/gestalt-core/src/main/java/org/github/gestalt/config/utils/ClassUtils.java @@ -275,4 +275,33 @@ public static Class primitiveToWrapper(final Class cls) { public static Class wrapperToPrimitive(final Class cls) { return wrapperPrimitiveMap.get(cls); } + + /** + * Returns whether the given {@code type} is a primitive or primitive wrapper ({@link Boolean}, {@link Byte}, + * {@link Character}, {@link Short}, {@link Integer}, {@link Long}, {@link Double}, {@link Float}). + * + * @param type The class to query or null. + * @return true if the given {@code type} is a primitive or primitive wrapper ({@link Boolean}, {@link Byte}, + * {@link Character}, {@link Short}, {@link Integer}, {@link Long}, {@link Double}, {@link Float}). + * @since 3.1 + */ + public static boolean isPrimitiveOrWrapper(final Class type) { + if (type == null) { + return false; + } + return type.isPrimitive() || isPrimitiveWrapper(type); + } + + /** + * Returns whether the given {@code type} is a primitive wrapper ({@link Boolean}, {@link Byte}, {@link Character}, + * {@link Short}, {@link Integer}, {@link Long}, {@link Double}, {@link Float}). + * + * @param type The class to query or null. + * @return true if the given {@code type} is a primitive wrapper ({@link Boolean}, {@link Byte}, {@link Character}, + * {@link Short}, {@link Integer}, {@link Long}, {@link Double}, {@link Float}). + * @since 3.1 + */ + public static boolean isPrimitiveWrapper(final Class type) { + return wrapperPrimitiveMap.containsKey(type); + } } diff --git a/gestalt-core/src/test/java/org/github/gestalt/config/decoder/MapDecoderTest.java b/gestalt-core/src/test/java/org/github/gestalt/config/decoder/MapDecoderTest.java index 1b526f5f0..5e477ea83 100644 --- a/gestalt-core/src/test/java/org/github/gestalt/config/decoder/MapDecoderTest.java +++ b/gestalt-core/src/test/java/org/github/gestalt/config/decoder/MapDecoderTest.java @@ -12,10 +12,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; class MapDecoderTest { @@ -84,6 +81,148 @@ void decode() { Assertions.assertEquals(6000, results.get("password")); } + + @Test + void decodeInt() { + + Map configs = new HashMap<>(); + configs.put("1", new LeafNode("100")); + configs.put("2", new LeafNode("300")); + configs.put("3", new LeafNode("6000")); + + MapDecoder decoder = new MapDecoder(); + + ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), new TypeCapture>() {}, + registry); + Assertions.assertTrue(validate.hasResults()); + Assertions.assertFalse(validate.hasErrors()); + + Map results = (Map) validate.results(); + Assertions.assertEquals(100, results.get(1)); + Assertions.assertEquals(300, results.get(2)); + Assertions.assertEquals(6000, results.get(3)); + } + + + static class User { + + private String name; + private Integer age; + + public User() { + + } + + public User(String name, Integer age) { + this.name = name; + this.age = age; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getAge() { + return age; + } + + public void setAge(Integer age) { + this.age = age; + } + } + + @Test + void decodeClass() { + Map user1 = new HashMap<>(); + user1.put("name", new LeafNode("steve")); + user1.put("age", new LeafNode("52")); + Map user2 = new HashMap<>(); + user2.put("name", new LeafNode("john")); + user2.put("age", new LeafNode("23")); + + Map configs = new HashMap<>(); + configs.put("user1", new MapNode(user1)); + configs.put("user2", new MapNode(user2)); + + MapDecoder decoder = new MapDecoder(); + + ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), new TypeCapture>() {}, + registry); + Assertions.assertTrue(validate.hasResults()); + Assertions.assertFalse(validate.hasErrors()); + + Map results = (Map) validate.results(); + Assertions.assertEquals("steve", results.get("user1").name); + Assertions.assertEquals(52, results.get("user1").age); + Assertions.assertEquals("john", results.get("user2").name); + Assertions.assertEquals(23, results.get("user2").age); + } + + @Test + void decodeNestedMap() { + Map retrySetting = new HashMap<>(); + retrySetting.put("times", new LeafNode("2")); + retrySetting.put("delay", new LeafNode("7")); + + Map settings = new HashMap<>(); + settings.put("timeout", new LeafNode("123")); + settings.put("retry", new MapNode(retrySetting)); + + Map configs = new HashMap<>(); + configs.put("port", new LeafNode("100")); + configs.put("uri", new LeafNode("300")); + configs.put("settings", new MapNode(settings)); + + MapDecoder decoder = new MapDecoder(); + + ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), new TypeCapture>() {}, + registry); + Assertions.assertTrue(validate.hasResults()); + Assertions.assertFalse(validate.hasErrors()); + + Map results = (Map) validate.results(); + Assertions.assertEquals(100, results.get("port")); + Assertions.assertEquals(300, results.get("uri")); + Assertions.assertEquals(123, results.get("settings.timeout")); + Assertions.assertEquals(2, results.get("settings.retry.times")); + Assertions.assertEquals(7, results.get("settings.retry.delay")); + } + + @Test + void decodeNestedMapWithArray() { + List retrySetting = new ArrayList<>(); + retrySetting.add(new LeafNode("2")); + retrySetting.add(new LeafNode("7")); + + Map settings = new HashMap<>(); + settings.put("timeout", new LeafNode("123")); + settings.put("retryList", new ArrayNode(retrySetting)); + + Map configs = new HashMap<>(); + configs.put("port", new LeafNode("100")); + configs.put("uri", new LeafNode("300")); + configs.put("settings", new MapNode(settings)); + + MapDecoder decoder = new MapDecoder(); + + ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), new TypeCapture>() {}, + registry); + Assertions.assertTrue(validate.hasResults()); + Assertions.assertFalse(validate.hasErrors()); + + Map results = (Map) validate.results(); + Assertions.assertEquals(100, results.get("port")); + Assertions.assertEquals(300, results.get("uri")); + Assertions.assertEquals(123, results.get("settings.timeout")); + Assertions.assertEquals(2, results.get("settings.retryList[0]")); + Assertions.assertEquals(7, results.get("settings.retryList[1]")); + } + + @Test void decodeNullLeafNodeValue() { MapDecoder decoder = new MapDecoder(); @@ -94,8 +233,7 @@ void decodeNullLeafNodeValue() { configs.put("password", new LeafNode("pass")); ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), - new TypeCapture>() { - }, registry); + new TypeCapture>() {}, registry); Assertions.assertTrue(validate.hasResults()); Assertions.assertTrue(validate.hasErrors()); @@ -121,8 +259,7 @@ void decodeNullKeyNodeValue() { configs.put("password", new LeafNode("pass")); ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), - new TypeCapture>() { - }, registry); + new TypeCapture>() {}, registry); Assertions.assertTrue(validate.hasResults()); Assertions.assertTrue(validate.hasErrors()); @@ -146,8 +283,7 @@ void decodeNullLeafNode() { configs.put("password", new LeafNode("pass")); ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), - new TypeCapture>() { - }, registry); + new TypeCapture>() {}, registry); Assertions.assertTrue(validate.hasResults()); Assertions.assertTrue(validate.hasErrors()); @@ -168,8 +304,7 @@ void decodeWrongNodeType() { MapDecoder decoder = new MapDecoder(); ValidateOf> validate = decoder.decode("db.host", new LeafNode("mysql.com"), - new TypeCapture>() { - }, registry); + new TypeCapture>() {}, registry); Assertions.assertFalse(validate.hasResults()); Assertions.assertTrue(validate.hasErrors()); @@ -188,9 +323,7 @@ void decodeWrongType() { MapDecoder decoder = new MapDecoder(); - ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), new TypeCapture>() { - }, - registry); + ValidateOf> validate = decoder.decode("db.host", new MapNode(configs), new TypeCapture>() {}, registry); Assertions.assertFalse(validate.hasResults()); Assertions.assertTrue(validate.hasErrors()); @@ -204,9 +337,7 @@ void decodeWrongType() { void decodeMapNodeNullInside() { MapDecoder decoder = new MapDecoder(); - ValidateOf> validate = decoder.decode("db.host", new MapNode(null), new TypeCapture>() { - }, - registry); + ValidateOf> validate = decoder.decode("db.host", new MapNode(null), new TypeCapture>() {}, registry); Assertions.assertFalse(validate.hasResults()); Assertions.assertTrue(validate.hasErrors()); diff --git a/gestalt-examples/gestalt-sample-java-latest/build.gradle.kts b/gestalt-examples/gestalt-sample-java-latest/build.gradle.kts index 7f317d2f9..f63845ac8 100644 --- a/gestalt-examples/gestalt-sample-java-latest/build.gradle.kts +++ b/gestalt-examples/gestalt-sample-java-latest/build.gradle.kts @@ -17,23 +17,20 @@ java { } testing { - suites { - val test by getting(JvmTestSuite::class) { - useJUnitJupiter() - testType.set(TestSuiteType.UNIT_TEST) - dependencies { - implementation(project(":gestalt-aws")) - implementation(project(":gestalt-core")) - implementation(project(":gestalt-hocon")) - implementation(project(":gestalt-kotlin")) - implementation(project(":gestalt-json")) - implementation(project(":gestalt-toml")) - implementation(project(":gestalt-yaml")) - - implementation(libs.aws.mock) - } + suites { + val test by getting(JvmTestSuite::class) { + useJUnitJupiter() + testType.set(TestSuiteType.UNIT_TEST) + dependencies { + implementation(project(":gestalt-core")) + implementation(project(":gestalt-hocon")) + implementation(project(":gestalt-kotlin")) + implementation(project(":gestalt-json")) + implementation(project(":gestalt-toml")) + implementation(project(":gestalt-yaml")) + } + } } - } } tasks.jar { diff --git a/gestalt-examples/gestalt-sample/build.gradle.kts b/gestalt-examples/gestalt-sample/build.gradle.kts index 29940bf4b..b96c19c0d 100644 --- a/gestalt-examples/gestalt-sample/build.gradle.kts +++ b/gestalt-examples/gestalt-sample/build.gradle.kts @@ -3,59 +3,64 @@ plugins { id("gestalt.java-test-conventions") id("gestalt.kotlin-common-conventions") id("gestalt.kotlin-test-conventions") - `jvm-test-suite` + `jvm-test-suite` } repositories { mavenLocal() mavenCentral() } + dependencies { - testImplementation(project(mapOf("path" to ":gestalt-google-cloud"))) + testImplementation(platform(libs.aws.bom)) + testImplementation(libs.aws.s3) + testImplementation(libs.aws.secret) + testImplementation(libs.aws.url.client) } - testing { - suites { - val test by getting(JvmTestSuite::class) { - useJUnitJupiter() - targets { - all { - testTask.configure { - options { - val junitOptions = this as JUnitPlatformOptions; - junitOptions.excludeTags = setOf("cloud") + suites { + val test by getting(JvmTestSuite::class) { + useJUnitJupiter() + targets { + all { + testTask.configure { + options { + val junitOptions = this as JUnitPlatformOptions; + junitOptions.excludeTags = setOf("cloud") + } + } + } } - } - } - } - testType.set(TestSuiteType.UNIT_TEST) - dependencies { - implementation(project(":gestalt-aws")) - implementation(project(":gestalt-core")) - implementation(project(":gestalt-hocon")) - implementation(project(":gestalt-kotlin")) - implementation(project(":gestalt-json")) - implementation(project(":gestalt-toml")) - implementation(project(":gestalt-yaml")) + testType.set(TestSuiteType.UNIT_TEST) + dependencies { + implementation(project(":gestalt-aws")) + implementation(project(":gestalt-core")) + implementation(project(":gestalt-hocon")) + implementation(project(":gestalt-kotlin")) + implementation(project(":gestalt-json")) + implementation(project(":gestalt-toml")) + implementation(project(":gestalt-yaml")) - implementation(project(":gestalt-google-cloud")) + implementation(project(":gestalt-google-cloud")) - implementation(project(":gestalt-kodein-di")) - implementation(libs.kodein.di) + implementation(project(":gestalt-kodein-di")) + implementation(libs.kodein.di) - implementation(project(":gestalt-koin-di")) - implementation(libs.koin.di) + implementation(project(":gestalt-koin-di")) + implementation(libs.koin.di) - implementation(libs.aws.mock) + implementation(libs.aws.mock) - implementation(project(":gestalt-guice")) - implementation(libs.guice) - } + + implementation(project(":gestalt-guice")) + implementation(libs.guice) + implementation(libs.testcontainers.junit5) + } + } } - } } diff --git a/gestalt-examples/gestalt-sample/src/test/java/org/github/gestalt/config/integration/GestaltSample.java b/gestalt-examples/gestalt-sample/src/test/java/org/github/gestalt/config/integration/GestaltSample.java index 8516b87c7..2abca08fc 100644 --- a/gestalt-examples/gestalt-sample/src/test/java/org/github/gestalt/config/integration/GestaltSample.java +++ b/gestalt-examples/gestalt-sample/src/test/java/org/github/gestalt/config/integration/GestaltSample.java @@ -1,6 +1,7 @@ package org.github.gestalt.config.integration; -import com.adobe.testing.s3mock.junit5.S3MockExtension; + +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; import com.google.inject.Guice; import com.google.inject.Injector; import org.github.gestalt.config.Gestalt; @@ -22,32 +23,71 @@ import org.github.gestalt.config.source.*; import org.github.gestalt.config.tag.Tags; import org.junit.jupiter.api.*; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.AttributeMap; import java.io.File; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; +@Testcontainers public class GestaltSample { - @RegisterExtension - static final S3MockExtension S3_MOCK = - S3MockExtension.builder().silent().withSecureConnection(false).build(); private static final String BUCKET_NAME = "testbucket"; + private static final String BUCKET_NAME_2 = "testbucket2"; + + private static final String S3MOCK_VERSION = System.getProperty("s3mock.version", "latest"); + private static final Collection INITIAL_BUCKET_NAMES = asList(BUCKET_NAME, BUCKET_NAME_2); + private static final String TEST_ENC_KEYREF = + "arn:aws:kms:us-east-1:1234567890:key/valid-test-key-ref"; + private static final String UPLOAD_FILE_NAME = "src/test/resources/default.properties"; - private final S3Client s3Client = S3_MOCK.createS3ClientV2(); + + private S3Client s3Client; + + @Container + private static final S3MockContainer s3Mock = + new S3MockContainer(S3MOCK_VERSION) + .withValidKmsKeys(TEST_ENC_KEYREF) + .withInitialBuckets(String.join(",", INITIAL_BUCKET_NAMES)); + + @BeforeEach + void setUp() { + // Must create S3Client after S3MockContainer is started, otherwise we can't request the random + // locally mapped port for the endpoint + var endpoint = s3Mock.getHttpsEndpoint(); + s3Client = createS3ClientV2(endpoint); + } + + protected S3Client createS3ClientV2(String endpoint) { + return S3Client.builder() + .region(Region.of("us-east-1")) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create("foo", "bar"))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .endpointOverride(URI.create(endpoint)) + .httpClient(UrlConnectionHttpClient.builder().buildWithDefaults( + AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, Boolean.TRUE).build())) + .build(); + } @BeforeAll public static void beforeAll() { @@ -543,7 +583,6 @@ public void integrationS3Test() throws GestaltException { final File uploadFile = new File(UPLOAD_FILE_NAME); - s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); s3Client.putObject( PutObjectRequest.builder().bucket(BUCKET_NAME).key(uploadFile.getName()).build(), RequestBody.fromFile(uploadFile)); @@ -704,6 +743,21 @@ private void validateResults(Gestalt gestalt) throws GestaltException { Assertions.assertEquals(25, pool.idleTimeoutSec); Assertions.assertEquals(33.0F, pool.defaultWait); + Map httpPoolMap = gestalt.getConfig("http.pool", new TypeCapture<>() { }); + + Assertions.assertEquals(50, httpPoolMap.get("maxperroute")); + Assertions.assertEquals(6000, httpPoolMap.get("validateafterinactivity")); + Assertions.assertEquals(60000, httpPoolMap.get("keepalivetimeoutms")); + Assertions.assertEquals(25, httpPoolMap.get("idletimeoutsec")); + + Map poolMap = gestalt.getConfig("http", new TypeCapture<>() { }); + + Assertions.assertEquals(50, poolMap.get("pool.maxperroute")); + Assertions.assertEquals(6000, poolMap.get("pool.validateafterinactivity")); + Assertions.assertEquals(60000, poolMap.get("pool.keepalivetimeoutms")); + Assertions.assertEquals(25, poolMap.get("pool.idletimeoutsec")); + + long startTime = System.nanoTime(); gestalt.getConfig("db", DataBase.class); long timeTaken = System.nanoTime() - startTime; diff --git a/gestalt-google-cloud/build.gradle.kts b/gestalt-google-cloud/build.gradle.kts index fd78c3d85..b582c84ee 100644 --- a/gestalt-google-cloud/build.gradle.kts +++ b/gestalt-google-cloud/build.gradle.kts @@ -1,18 +1,19 @@ plugins { - id("gestalt.java-library-conventions") - id("gestalt.java-test-conventions") - id("gestalt.java-code-quality-conventions") - id("gestalt.java-publish-conventions") + id("gestalt.java-library-conventions") + id("gestalt.java-test-conventions") + id("gestalt.java-code-quality-conventions") + id("gestalt.java-publish-conventions") } dependencies { - implementation(project(":gestalt-core")) - implementation(libs.google.storage) - implementation(libs.google.secret) + implementation(project(":gestalt-core")) + implementation(platform(libs.google.libraries)) + implementation(libs.google.storage) + implementation(libs.google.secret) } tasks.jar { - manifest { - attributes("Automatic-Module-Name" to "org.github.gestalt.google") - } + manifest { + attributes("Automatic-Module-Name" to "org.github.gestalt.google") + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 10ccbf6b3..e42a3f7c8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,9 +19,8 @@ weldCore = "4.0.3.Final" jackson = "2.15.2" hocon = "1.4.2" # Cloud -aws = "2.20.99" -gcpStorage = "2.23.0" -gcpSecret = "2.20.0" +awsBom = "2.20.56" +gcpLibraries = "26.18.0" # Git support jgit = "6.6.0.202305301015-r" eddsa = "0.3.0" @@ -31,7 +30,9 @@ assertJ = "3.24.2" mockito = "5.2.0" mockk = "1.13.5" koTestAssertions = "5.6.2" +# @pin last version to support jdk 11 awsMock = "2.17.0" +testcontainers = "1.18.3" # static code analysis errorprone = "2.20.0" gradleErrorProne = "3.1.0" @@ -72,11 +73,13 @@ jackson-toml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-t jackson-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" } hocon = { module = "com.typesafe:config", version.ref = "hocon" } # Cloud -aws-S3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } -aws-secret = { module = "software.amazon.awssdk:secretsmanager", version.ref = "aws" } -aws-url-client = { module = "software.amazon.awssdk:url-connection-client", version.ref = "aws" } -google-storage = { module = "com.google.cloud:google-cloud-storage", version.ref = "gcpStorage" } -google-secret = { module = "com.google.cloud:google-cloud-secretmanager", version.ref = "gcpSecret" } +aws-bom = { module = "software.amazon.awssdk:bom", version.ref = "awsBom" } +aws-S3 = { module = "software.amazon.awssdk:s3" } +aws-secret = { module = "software.amazon.awssdk:secretsmanager" } +aws-url-client = { module = "software.amazon.awssdk:url-connection-client" } +google-libraries = { module = "com.google.cloud:libraries-bom", version.ref = "gcpLibraries" } +google-storage = { module = "com.google.cloud:google-cloud-storage" } +google-secret = { module = "com.google.cloud:google-cloud-secretmanager" } # Git Support jgit = { module = "org.eclipse.jgit:org.eclipse.jgit", version.ref = "jgit" } jgit-apache-SSH = { module = "org.eclipse.jgit:org.eclipse.jgit.ssh.apache", version.ref = "jgit" } @@ -88,7 +91,8 @@ assertJ = { module = "org.assertj:assertj-core", version.ref = "assertJ" } mockito = { module = "org.mockito:mockito-inline", version.ref = "mockito" } mockk = { module = "io.mockk:mockk", version.ref = "mockk" } koTestAssertions = { module = "io.kotest:kotest-assertions-core-jvm", version.ref = "koTestAssertions" } -aws-Mock = { module = "com.adobe.testing:s3mock-junit5", version.ref = "awsMock" } +aws-Mock = { module = "com.adobe.testing:s3mock-testcontainers", version.ref = "awsMock" } +testcontainers-junit5 = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } # Benchmarking jmh = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } jmh-annotations = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } @@ -115,11 +119,11 @@ gestalt-koin-di = { module = "com.github.gestalt-config:gestalt-koin-di", versio [bundles] jackson = [ - "jackson-core", - "jackson-databind", - "jackson-dataformat-yaml", - "jackson-java8", - "jackson-jsr310", + "jackson-core", + "jackson-databind", + "jackson-dataformat-yaml", + "jackson-java8", + "jackson-jsr310", ] [plugins]