Feast 0.3 Continuous Integration (CI) Update (feast-dev#271)

* Update prow config and test script so it works for Feast 0.3

* Update CI test script for golang to use correct path

* Set pipefail option so test exit code is correct
For those commands that pipe the test output

* Update group id and version grpc-spring-boot-starter dependency
- Newer version is hosted on Maven central which is more maintained than other repo

* Add JVM heap settings for Maven surefire
Otherwise it fails to run some tests due to insufficient memory

* Fix potential setup/teardown error when running ImportJobTest
- Set project to empty (DataflowOptions in Beam require project to be not null)
- Ignore error when shutting down Kafka server during tear down. This should not affect test result.

* Update remote URI to download cached Maven packages
- This tar archive contains packages used by Feast 0.3 rather than previous version of Feast

* Fix path to logs artifacts

* Update maven enforcer rule for Maven and JDK versions
- Allow more ranges of versions as long as they are not breaking changes

* Use batch mode when initializing Maven for cleaner build log

* Skip maven enforcer when running test for specific project

* Fix incorrect order for java-sdk test script

* Install database and kafka dependencies for Feast end to end test

* Update Python SDK to not set source in FeatureSet when using default source
Update in progress script for integration test: TODO.sh

* Fix Python SDK type mapping
- float64 should map to ValueType.DOUBLE instead of ValueType.FLOAT
- Ensure Python SDK installation use protobuf v3.10 from pypi. Earlier version of protobuf do not support accessing Enum value in Proto object directly.
- Ensure version is passed as INT in e2e test. Set default allow_dirty to true so it's easier to debug error when running e2e test locally.

* Refactor prow scripts such that every test is defined in a separate script
- Since there are only 6 tests, there is not much benefit of combining tests into a single script
  Having each test defined on diff script file makes debugging easier

* Remove reference to actual project id

* Log current stage in end to end test script
- Update test case DirectRunnerJobManagerTest.shouldStartDirectJobAndRegisterPipelineResult to set project option to empty string for DirectRunner
- Update /usr/sbin/policy-rc.d used in Maven Docker image, so Redis installation will succeed

* Fix missing option to specify miniconda download target
- Also use cached maven packages when running e2e test

* Fix incorrect path to installing Python SDK

* Add small wait after ingestion, before validating the saved features, in feast e2e test

* Increase wait time before checking, after applying feature set

* Make log cleaner

.prow/config.yaml

@@ -21,7 +21,7 @@ plank:
 deck:
   tide_update_period: 1s
   spyglass:
-    size_limit: 100e+6 # 100MB
+    size_limit: 50e+6 # 50MB
     viewers:
       "started.json|finished.json": ["metadata"]
       "build-log.txt": ["buildlog"]
@@ -50,132 +50,56 @@ tide:
 # presubmits list Prow jobs that run on pull requests
 presubmits:
   gojek/feast:
-  - name: unit-test-core
+  - name: test-core-and-ingestion
     decorate: true
     always_run: true
     spec:
-      volumes:
-      - name: service-account
-        secret:
-          secretName: prow-service-account
       containers:
       - image: maven:3.6-jdk-8
-        volumeMounts:
-        - name: service-account
-          mountPath: /etc/service-account
-          readOnly: true
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/service-account/service-account.json
-        command: [".prow/scripts/run_unit_test.sh", "--component", "core"]
+        command: [".prow/scripts/test-core-ingestion.sh"]
 
-  - name: unit-test-ingestion
+  - name: test-serving
     decorate: true
     always_run: true
     spec:
-      volumes:
-      - name: service-account
-        secret:
-          secretName: prow-service-account
       containers:
       - image: maven:3.6-jdk-8
-        volumeMounts:
-        - name: service-account
-          mountPath: /etc/service-account
-          readOnly: true
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/service-account/service-account.json
-        command: [".prow/scripts/run_unit_test.sh", "--component", "ingestion"]
+        command: [".prow/scripts/test-serving.sh"]
 
-  - name: unit-test-serving
+  - name: test-java-sdk
     decorate: true
     always_run: true
     spec:
       containers:
       - image: maven:3.6-jdk-8
-        command: [".prow/scripts/run_unit_test.sh", "--component", "serving"]
+        command: [".prow/scripts/test-java-sdk.sh"]
 
-  - name: unit-test-cli
+  - name: test-python-sdk
     decorate: true
     always_run: true
     spec:
       containers:
-      - image: golang:1.12
-        env:
-        - name: GO111MODULE
-          value: "on"
-        command: [".prow/scripts/run_unit_test.sh", "--component", "cli"]
+      - image: python:3.7
+        command: [".prow/scripts/test-python-sdk.sh"]
 
-  - name: unit-test-python-sdk
+  - name: test-golang-sdk
     decorate: true
     always_run: true
     spec:
-      volumes:
-      - name: service-account
-        secret:
-          secretName: prow-service-account
       containers:
-      - image: python:3.6
-        volumeMounts:
-        - name: service-account
-          mountPath: /etc/service-account
-          readOnly: true
-        env:
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: /etc/service-account/service-account.json
-        command: [".prow/scripts/run_unit_test.sh", "--component", "python-sdk"]
+      - image: golang:1.13
+        command: [".prow/scripts/test-golang-sdk.sh"]
 
-  - name: integration-test
+  - name: test-end-to-end
     decorate: true
     always_run: true
     spec:
-      volumes:
-      - name: docker-socket-volume
-        hostPath:
-          path: /var/run/docker.sock
-          type: File
-      - name: service-account
-        secret:
-          secretName: prow-service-account
-      nodeSelector:
-        os: ubuntu
       containers:
-      - image: google/cloud-sdk
-        # securityContext and docker socket volume mounts are needed because we are building
-        # Docker images in this job
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: docker-socket-volume
-          mountPath: /var/run/docker.sock
-        - name: service-account
-          mountPath: /etc/service-account
-          readOnly: true
-        command:
-        - bash
-        - -c
-        - |
-          export FEAST_HOME=${PWD}
-          export FEAST_IMAGE_REGISTRY=us.gcr.io
-          export FEAST_IMAGE_TAG=${PULL_PULL_SHA}
-          export FEAST_WAREHOUSE_DATASET=feast_build_${BUILD_ID}
-          export FEAST_CORE_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80
-          export FEAST_SERVING_URL=build-${BUILD_ID:0:5}.drone.feast.ai:80
-          export FEAST_RELEASE_NAME=feast-${BUILD_ID:0:5}
-          export BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${BUILD_ID:0:5}/integration-tests/testdata/feature_values/ingestion_1.csv
-          export KAFKA_BROKERS=10.128.0.201:9092
-          export KAFKA_TOPICS=feast_build_${BUILD_ID:0:5}
-
-          . .prow/scripts/prepare_integration_test.sh
-          .prow/scripts/install_feast_and_run_e2e_test.sh
-          TEST_EXIT_CODE=$?
-          .prow/scripts/cleanup_feast_installation.sh
-          
-          exit ${TEST_EXIT_CODE}
+      - image: maven:3.6-jdk-8
+        command: [".prow/scripts/test-end-to-end.sh"]
 
 # TODO: do a release when a git tag is pushed
-# postsubmits list Prow jobs that run on every push 
 #
+# postsubmits list Prow jobs that run on every push 
 # postsubmits:
-#   gojek/feast:
+#   gojek/feast:

.prow/scripts/prepare_maven_cache.sh → .prow/scripts/download-maven-cache.sh

@@ -23,5 +23,11 @@ done
 if [[ ! ${ARCHIVE_URI} ]]; then usage; exit 1; fi
 if [[ ! ${OUTPUT_DIR}  ]]; then usage; exit 1; fi
 
+# Install Google Cloud SDK if gsutil command not exists
+if [[ ! $(command -v gsutil) ]]; then 
+  CURRENT_DIR=$(dirname "$BASH_SOURCE")
+  . "${CURRENT_DIR}"/install_google_cloud_sdk.sh
+fi  
+
 gsutil -q cp ${ARCHIVE_URI} /tmp/.m2.tar
 tar xf /tmp/.m2.tar -C ${OUTPUT_DIR}

.prow/scripts/install_google_cloud_sdk.sh

@@ -23,14 +23,14 @@ while [ "$1" != "" ]; do
   shift
 done
 
-GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz
+GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-266.0.0-linux-x86_64.tar.gz
 GOOGLE_PROJECT_ID=kf-feast
 KUBE_CLUSTER_NAME=primary-test-cluster
 KUBE_CLUSTER_ZONE=us-central1-a
 
 curl -s ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz -C /
 export PATH=/google-cloud-sdk/bin:${PATH}
-gcloud -q components install kubectl
+gcloud -q components install kubectl &> /var/log/kubectl.install.log
 
 if [[ ${KEY_FILE} ]]; then 
     gcloud -q auth activate-service-account --key-file=${KEY_FILE}