add java options to feast-serving helm chart

Signed-off-by: Oleksii Moskalenko <moskalenko.alexey@gmail.com>
feast-dev · Jan 28, 2021 · b595f0d · b595f0d
1 parent ed6389e
commit b595f0d
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 31 deletions.
diff --git a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml
@@ -92,6 +92,9 @@ spec:
 
         command:
         - java
+        {{- range $opt := .Values.JAVA_OPTIONS }}
+        - {{ $opt }}
+        {{- end }}
         - -jar
         - /opt/feast/feast-serving.jar
         - --spring.config.location=
@@ -117,7 +120,11 @@ spec:
         {{- if .Values.livenessProbe.enabled }}
         livenessProbe:
           exec:
-            command: ["grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"]
+            command:
+            - "grpc-health-probe"
+            - "-addr=:{{ .Values.service.grpc.targetPort }}"
+            - "-connect-timeout={{ .Values.livenessProbe.timeoutSeconds }}s"
+            - "-rpc-timeout={{ .Values.livenessProbe.timeoutSeconds }}s"
           initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
           successThreshold: {{ .Values.livenessProbe.successThreshold }}
@@ -128,7 +135,11 @@ spec:
         {{- if .Values.readinessProbe.enabled }}
         readinessProbe:
           exec:
-            command: ["grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"]
+            command:
+            - "grpc-health-probe"
+            - "-addr=:{{ .Values.service.grpc.targetPort }}"
+            - "-connect-timeout={{ .Values.readinessProbe.timeoutSeconds }}s"
+            - "-rpc-timeout={{ .Values.readinessProbe.timeoutSeconds }}s"
           initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
           successThreshold: {{ .Values.readinessProbe.successThreshold }}

diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml
@@ -25,6 +25,8 @@ application-secret.yaml:
 application-override.yaml:
   enabled: true
 
+JAVA_OPTIONS: []
+
 gcpServiceAccount:
   # gcpServiceAccount.enabled -- Flag to use [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) JSON key
   # Cloud service account JSON key file.

diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile
@@ -42,12 +42,12 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa
 # Build stage 2: Production
 # ============================================================
 
-FROM openjdk:11-jre-slim as production
+FROM amazoncorretto:11 as production
 ARG VERSION=dev
 COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar
 COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe
 CMD ["java",\
-     "-Xms1024m",\
-     "-Xmx1024m",\
+     "-Xms1g",\
+     "-Xmx4g",\
      "-jar",\
      "/opt/feast/feast-serving.jar"]
diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java
@@ -103,15 +103,15 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re
             .map(r -> getMetadataMap(r.getFieldsMap(), false, false))
             .collect(Collectors.toList());
 
-    Span onlineRetrievalSpan = tracer.buildSpan("onlineRetrieval").start();
-    if (onlineRetrievalSpan != null) {
-      onlineRetrievalSpan.setTag("entities", entityRows.size());
-      onlineRetrievalSpan.setTag("features", featureReferences.size());
+    Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start();
+    if (storageRetrievalSpan != null) {
+      storageRetrievalSpan.setTag("entities", entityRows.size());
+      storageRetrievalSpan.setTag("features", featureReferences.size());
     }
     List<List<Feature>> entityRowsFeatures =
         retriever.getOnlineFeatures(projectName, entityRows, featureReferences);
-    if (onlineRetrievalSpan != null) {
-      onlineRetrievalSpan.finish();
+    if (storageRetrievalSpan != null) {
+      storageRetrievalSpan.finish();
     }
 
     if (entityRowsFeatures.size() != entityRows.size()) {
@@ -145,6 +145,8 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re
                       }
                     }));
 
+    Span postProcessingSpan = tracer.buildSpan("postProcessing").start();
+
     for (int i = 0; i < entityRows.size(); i++) {
       GetOnlineFeaturesRequestV2.EntityRow entityRow = entityRows.get(i);
       List<Feature> curEntityRowFeatures = entityRowsFeatures.get(i);
@@ -198,6 +200,11 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re
         }
       }
     }
+
+    if (postProcessingSpan != null) {
+      postProcessingSpan.finish();
+    }
+
     populateHistogramMetrics(entityRows, featureReferences, projectName);
     populateFeatureCountMetrics(featureReferences, projectName);
 
@@ -339,29 +346,23 @@ private void populateHistogramMetrics(
    */
   private void populateCountMetrics(
       Map<String, GetOnlineFeaturesResponse.FieldStatus> statusMap, String project) {
-    statusMap
-        .entrySet()
-        .forEach(
-            es -> {
-              String featureRefString = es.getKey();
-              GetOnlineFeaturesResponse.FieldStatus status = es.getValue();
-              if (status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND) {
-                Metrics.notFoundKeyCount.labels(project, featureRefString).inc();
-              }
-              if (status == GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE) {
-                Metrics.staleKeyCount.labels(project, featureRefString).inc();
-              }
-            });
+    statusMap.forEach(
+        (featureRefString, status) -> {
+          if (status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND) {
+            Metrics.notFoundKeyCount.labels(project, featureRefString).inc();
+          }
+          if (status == GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE) {
+            Metrics.staleKeyCount.labels(project, featureRefString).inc();
+          }
+        });
   }
 
   private void populateFeatureCountMetrics(
       List<FeatureReferenceV2> featureReferences, String project) {
-    featureReferences
-        .parallelStream()
-        .forEach(
-            featureReference ->
-                Metrics.requestFeatureCount
-                    .labels(project, FeatureV2.getFeatureStringRef(featureReference))
-                    .inc());
+    featureReferences.forEach(
+        featureReference ->
+            Metrics.requestFeatureCount
+                .labels(project, FeatureV2.getFeatureStringRef(featureReference))
+                .inc());
   }
 }