From f9f14ead51a0cf2c70deb4f1df4ecdbef9a9f8cc Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 4 Jan 2022 16:42:35 +0100 Subject: [PATCH 1/2] support spring-like properties paths Signed-off-by: pyalex --- .../feature-server/templates/deployment.yaml | 3 +- infra/docker-compose/docker-compose.yml | 4 +- .../serving/ServingGuiceApplication.java | 2 +- .../serving/config/ApplicationProperties.java | 18 +++++++-- .../config/ApplicationPropertiesModule.java | 37 +++++++++++++++++-- .../feast/serving/config/ServerModule.java | 3 +- 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/infra/charts/feast/charts/feature-server/templates/deployment.yaml b/infra/charts/feast/charts/feature-server/templates/deployment.yaml index 9327747423..ad0529978d 100644 --- a/infra/charts/feast/charts/feature-server/templates/deployment.yaml +++ b/infra/charts/feast/charts/feature-server/templates/deployment.yaml @@ -89,8 +89,7 @@ spec: - java - -jar - /opt/feast/feast-serving.jar - - --spring.config.location= - {{- if index .Values "application.yaml" "enabled" -}} + - {{- if index .Values "application.yaml" "enabled" -}} classpath:/application.yml {{- end }} {{- if index .Values "application-generated.yaml" "enabled" -}} diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 98131d6ccf..579dc6d65f 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -16,7 +16,7 @@ services: - java - -jar - /opt/feast/feast-core.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + - classpath:/application.yml,file:/etc/feast/application.yml jobservice: image: gcr.io/kf-feast/feast-jobservice:${FEAST_VERSION} @@ -104,7 +104,7 @@ services: - java - -jar - /opt/feast/feast-serving.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml + - classpath:/application.yml,file:/etc/feast/application.yml redis: image: redis:5-alpine diff --git a/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java b/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java index 224c3e8e55..664d6dd4ec 100644 --- a/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java +++ b/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java @@ -27,7 +27,7 @@ public class ServingGuiceApplication { public static void main(String[] args) throws InterruptedException, IOException { if (args.length == 0) { throw new RuntimeException( - "Path to application configuration file needs to be specifed via CLI"); + "Path to application configuration file needs to be specified via CLI"); } final Injector i = diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java b/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java index 4d822d8dbc..c79abf3a96 100644 --- a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java +++ b/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java @@ -21,6 +21,8 @@ // https://www.baeldung.com/configuration-properties-in-spring-boot // https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-typesafe-configuration-properties +import com.fasterxml.jackson.annotation.JsonMerge; +import com.fasterxml.jackson.annotation.OptBoolean; import feast.common.logging.config.LoggingProperties; import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig; import feast.storage.connectors.redis.retriever.RedisStoreConfig; @@ -83,6 +85,7 @@ public void setActiveStore(String activeStore) { /** * Collection of store configurations. The active store is selected by the "activeStore" field. */ + @JsonMerge(OptBoolean.FALSE) private List stores = new ArrayList<>(); /* Metric tracing properties. */ @@ -177,6 +180,9 @@ public static class Store { private Map config = new HashMap<>(); + // empty construct for deserialization + public Store() {} + public Store(String name, String type, Map config) { this.name = name; this.type = type; @@ -210,6 +216,10 @@ public StoreType getType() { return StoreType.valueOf(this.type); } + public void setType(String type) { + this.type = type; + } + /** * Gets the configuration to this specific store. This is a map of strings. These options are * unique to the store. Please see protos/feast/core/Store.proto for the store specific @@ -217,10 +227,6 @@ public StoreType getType() { * * @return Returns the store specific configuration */ - public Map getConfig() { - return config; - } - public RedisClusterStoreConfig getRedisClusterConfig() { return new RedisClusterStoreConfig( this.config.get("connection_string"), @@ -235,6 +241,10 @@ public RedisStoreConfig getRedisConfig() { Boolean.valueOf(this.config.getOrDefault("ssl", "false")), this.config.getOrDefault("password", "")); } + + public void setConfig(Map config) { + this.config = config; + } } public static class Server { diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java b/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java index f5a542137c..07183fc710 100644 --- a/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java +++ b/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java @@ -17,12 +17,15 @@ package feast.serving.config; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.io.Resources; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.Singleton; -import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; public class ApplicationPropertiesModule extends AbstractModule { private final String[] args; @@ -36,9 +39,37 @@ public ApplicationPropertiesModule(String[] args) { public ApplicationProperties provideApplicationProperties() throws IOException { ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); mapper.findAndRegisterModules(); - ApplicationProperties properties = - mapper.readValue(new File(this.args[0]), ApplicationProperties.class); + mapper.setDefaultMergeable(Boolean.TRUE); + + ApplicationProperties properties = new ApplicationProperties(); + ObjectReader objectReader = mapper.readerForUpdating(properties); + + String[] filePaths = this.args[0].split(","); + for (String filePath : filePaths) { + objectReader.readValue(readPropertiesFile(filePath)); + } return properties; } + + /** + * Read file path in spring compatible format, eg classpath:/application.yml or + * file:/path/application.yml + */ + private byte[] readPropertiesFile(String filePath) throws IOException { + if (filePath.startsWith("classpath:")) { + filePath = filePath.substring("classpath:".length()); + if (filePath.startsWith("/")) { + filePath = filePath.substring(1); + } + + return Resources.toByteArray(Resources.getResource(filePath)); + } + + if (filePath.startsWith("file")) { + filePath = filePath.substring("file:".length()); + } + + return Files.readAllBytes(Path.of(filePath)); + } } diff --git a/java/serving/src/main/java/feast/serving/config/ServerModule.java b/java/serving/src/main/java/feast/serving/config/ServerModule.java index 6993857935..cb3a18cf95 100644 --- a/java/serving/src/main/java/feast/serving/config/ServerModule.java +++ b/java/serving/src/main/java/feast/serving/config/ServerModule.java @@ -17,6 +17,7 @@ package feast.serving.config; import com.google.inject.AbstractModule; +import com.google.inject.Provides; import feast.serving.grpc.OnlineServingGrpcServiceV2; import io.grpc.Server; import io.grpc.ServerBuilder; @@ -30,7 +31,7 @@ protected void configure() { bind(OnlineServingGrpcServiceV2.class); } - // @Provides + @Provides public Server provideGrpcServer( ApplicationProperties applicationProperties, OnlineServingGrpcServiceV2 onlineServingGrpcServiceV2, From f2a4fe2c5c6a1e93dcb755df7b9a1d73060ef15a Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 4 Jan 2022 16:49:35 +0100 Subject: [PATCH 2/2] typo Signed-off-by: pyalex --- .../main/java/feast/serving/config/ApplicationProperties.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java b/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java index c79abf3a96..c75ca98451 100644 --- a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java +++ b/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java @@ -180,7 +180,7 @@ public static class Store { private Map config = new HashMap<>(); - // empty construct for deserialization + // default construct for deserialization public Store() {} public Store(String name, String type, Map config) {