Feast 0.3 Continuous Integration (CI) Update

.prow/config.yaml

@@ -50,7 +50,7 @@ tide:
 # presubmits list Prow jobs that run on pull requests
 presubmits:
   gojek/feast:
-  - name: unit-test-core
+  - name: test-core-and-ingestion
     decorate: true
     always_run: true
     spec:
@@ -67,66 +67,41 @@ presubmits:
         env:
         - name: GOOGLE_APPLICATION_CREDENTIALS
           value: /etc/service-account/service-account.json
-        command: [".prow/scripts/run_unit_test.sh", "--component", "core"]
+        command: [".prow/scripts/run_test.sh", "--component", "core-ingestion"]
 
-  - name: unit-test-ingestion
+  - name: test-serving
     decorate: true
     always_run: true
     spec:
-      volumes:
-      - name: service-account
-        secret:
-          secretName: prow-service-account
       containers:
       - image: maven:3.6-jdk-8
-        volumeMounts:
-        - name: service-account
-          mountPath: /etc/service-account
-          readOnly: true
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/service-account/service-account.json
-        command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"]
+        command: [".prow/scripts/run_test.sh", "--component", "serving"]
 
-  - name: unit-test-serving
+  - name: test-java-sdk
     decorate: true
     always_run: true
     spec:
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/run_unit_test.sh", "--component", "serving"]
+        command: [".prow/scripts/run_test.sh", "--component", "java-sdk"]
 
-  - name: unit-test-cli
+  - name: test-python-sdk
     decorate: true
     always_run: true
     spec:
       containers:
-      - image: golang:1.12
-        env:
-        - name: GO111MODULE
-          value: "on"
-        command: [".prow/scripts/run_unit_test.sh", "--component", "cli"]
+      - image: python:3.6
+        command: [".prow/scripts/run_test.sh", "--component", "python-sdk"]
 
-  - name: unit-test-python-sdk
+  - name: test-golang-sdk
     decorate: true
     always_run: true
     spec:
-      volumes:
-      - name: service-account
-        secret:
-          secretName: prow-service-account
       containers:
-      - image: python:3.6
-        volumeMounts:
-        - name: service-account
-          mountPath: /etc/service-account
-          readOnly: true
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/service-account/service-account.json
-        command: [".prow/scripts/run_unit_test.sh", "--component", "python-sdk"]
+      - image: golang:1.13
+        command: [".prow/scripts/run_test.sh", "--component", "golang-sdk"]
 
-  - name: integration-test
+  - name: test-end-to-end
     decorate: true
     always_run: true
     spec:
@@ -175,7 +150,7 @@ presubmits:
           exit ${TEST_EXIT_CODE}
 
 # TODO: do a release when a git tag is pushed
-# postsubmits list Prow jobs that run on every push 
 #
+# postsubmits list Prow jobs that run on every push 
 # postsubmits:
-#   gojek/feast:
+#   gojek/feast:

.prow/scripts/install_google_cloud_sdk.sh

@@ -23,7 +23,7 @@ while [ "$1" != "" ]; do
   shift
 done
 
-GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz
+GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-266.0.0-linux-x86_64.tar.gz
 GOOGLE_PROJECT_ID=kf-feast
 KUBE_CLUSTER_NAME=primary-test-cluster
 KUBE_CLUSTER_ZONE=us-central1-a

.prow/scripts/prepare_maven_cache.sh

@@ -23,5 +23,11 @@ done
 if [[ ! ${ARCHIVE_URI} ]]; then usage; exit 1; fi
 if [[ ! ${OUTPUT_DIR}  ]]; then usage; exit 1; fi
 
+# Install Google Cloud SDK if gsutil command not exists
+if [[ ! $(command -v gsutil) ]]; then 
+  CURRENT_DIR=$(dirname "$BASH_SOURCE")
+  . "${CURRENT_DIR}"/install_google_cloud_sdk.sh
+fi  
+
 gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar
 tar xf /tmp/.m2.tar -C ${OUTPUT_DIR}

.prow/scripts/run_test.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# Default artifact location setting in Prow jobs
+LOGS_ARTIFACT_PATH=/logs/artifacts
+
+set -o pipefail
+
+usage()
+{
+    echo "Run test on a Feast component.
+
+Usage: run_test.sh --component <feast_component>
+
+<feast_component> is one of:
+- core-ingestion (core depends on ingestion so they are tested together)
+- serving 
+- java-sdk
+- python-sdk
+- golang-sdk
+
+This script also runs commands before and after the main test task, such as:
+- Download cached Maven packages for faster tests
+- Saving the test output report so it can be viewed with Spyglass UI in Prow.
+  By default, the configured log path is "/logs" and test artifacts should
+  be saved to "/logs/artifacts" directory.
+"
+}
+
+while [ "$1" != "" ]; do
+  case "$1" in
+      --component )       COMPONENT="$2";    shift;;
+      * )                 usage; exit 1
+  esac
+  shift
+done
+
+if [[ ! ${COMPONENT} ]]; then 
+  usage; exit 1; 
+fi
+
+if [[ ${COMPONENT} == "core-ingestion" ]]; then
+
+  .prow/scripts/prepare_maven_cache.sh \
+    --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/
+
+  # Core depends on Ingestion so they are tested together
+  mvn --define skipTests=true --projects core,ingestion clean install
+  mvn --projects core,ingestion test
+  TEST_EXIT_CODE=$?
+
+  mkdir -p ${LOGS_ARTIFACT_PATH}/surefire-reports
+  cp core/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/
+  cp ingestion/target/surefire-reports/* ${LOGS_ARTIFACT_PATH}/surefire-reports/
+
+elif [[ ${COMPONENT} == "serving" ]]; then
+
+  .prow/scripts/prepare_maven_cache.sh \
+    --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/
+
+  mvn --define skipTests=true --projects serving clean install
+  mvn --projects serving test
+  TEST_EXIT_CODE=$?
+
+  cp -r serving/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports
+
+elif [[ ${COMPONENT} == "java-sdk" ]]; then
+
+  .prow/scripts/prepare_maven_cache.sh \
+    --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar --output-dir /root/
+
+  mvn --define skipTests=true --projects sdk/java clean install
+  mvn --projects sdk/java test
+  TEST_EXIT_CODE=$?
+
+  cp -r sdk/java/target/surefire-reports ${LOGS_ARTIFACT_PATH}/surefire-reports
+
+elif [[ ${COMPONENT} == "python-sdk" ]]; then
+
+  cd sdk/python
+  pip install -r requirements-test.txt
+  pip install .
+  pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+  TEST_EXIT_CODE=$?
+
+elif [[ ${COMPONENT} == "golang-sdk" ]]; then
+
+  cd sdk/go
+  go test -v 2>&1 | tee /tmp/test_output
+  TEST_EXIT_CODE=$?
+
+  go get -u github.com/jstemmer/go-junit-report
+  cat /tmp/test_output | ${GOPATH}/bin/go-junit-report > ${LOGS_ARTIFACT_PATH}/golang-sdk-test-report.xml
+
+else
+  usage; exit 1
+fi
+
+exit ${TEST_EXIT_CODE}

core/pom.xml

@@ -102,9 +102,9 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-log4j2</artifactId>
         </dependency>
-        <!--compile 'org.lognet:grpc-spring-boot-starter:2.4.1'-->
+        <!--compile io.github.lognet:grpc-spring-boot-starter:3.0.2'-->
         <dependency>
-            <groupId>org.lognet</groupId>
+            <groupId>io.github.lognet</groupId>
             <artifactId>grpc-spring-boot-starter</artifactId>
         </dependency>
         <!--compile "org.springframework.boot:spring-boot-starter-data-jpa:${springBootVersion}"-->

ingestion/src/test/java/feast/ingestion/ImportJobTest.java

@@ -116,6 +116,7 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow()
     options.setStoreJson(
         Collections.singletonList(
             JsonFormat.printer().omittingInsignificantWhitespace().print(redis)));
+    options.setProject("");
     options.setBlockOnRun(false);
 
     int inputSize = 4096;

ingestion/src/test/java/feast/test/TestUtil.java

@@ -92,7 +92,11 @@ public static void start(String kafkaHost, int kafkaPort, short kafkaReplication
 
     public static void stop() {
       if (server != null) {
-        server.shutdown();
+        try {
+          server.shutdown();
+        } catch (Exception e) {
+          e.printStackTrace();
+        }
       }
     }
   }

pom.xml

@@ -250,9 +250,9 @@
             </dependency>
 
             <dependency>
-                <groupId>org.lognet</groupId>
+                <groupId>io.github.lognet</groupId>
                 <artifactId>grpc-spring-boot-starter</artifactId>
-                <version>2.4.1</version>
+                <version>3.0.2</version>
             </dependency>
 
             <!-- Logging -->
@@ -392,10 +392,10 @@
                         <configuration>
                             <rules>
                                 <requireMavenVersion>
-                                    <version>3.0.5</version>
+                                    <version>[3.5,4.0)</version>
                                 </requireMavenVersion>
                                 <requireJavaVersion>
-                                    <version>1.8.0</version>
+                                    <version>[1.8,1.9)</version>
                                 </requireJavaVersion>
                                 <reactorModuleConvergence />
                             </rules>
@@ -432,7 +432,7 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.22.1</version>
                 <configuration>
-                    <argLine>-Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
+                    <argLine>-Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true</argLine>
                     <excludes>
                         <groups>IntegrationTest</groups>
                     </excludes>

serving/pom.xml

@@ -100,9 +100,9 @@
       <optional>true</optional>
     </dependency>
 
-    <!--compile 'org.lognet:grpc-spring-boot-starter:2.4.1'-->
+    <!--compile io.github.lognet:grpc-spring-boot-starter:3.0.2'-->
     <dependency>
-      <groupId>org.lognet</groupId>
+      <groupId>io.github.lognet</groupId>
       <artifactId>grpc-spring-boot-starter</artifactId>
     </dependency>
     <!--compile "org.springframework.boot:spring-boot-starter-actuator:${springBootVersion}"-->