Parameterize end to end test scripts. (#433)

feast-dev · Mar 16, 2020 · 1517579 · 1517579
1 parent 517e4c6
commit 1517579
Show file tree

Hide file tree

Showing 2 changed files with 81 additions and 63 deletions.
diff --git a/.prow/scripts/test-end-to-end-batch.sh b/.prow/scripts/test-end-to-end-batch.sh
@@ -4,22 +4,32 @@ set -e
 set -o pipefail
 
 if ! cat /etc/*release | grep -q stretch; then
-    echo ${BASH_SOURCE} only supports Debian stretch. 
+    echo ${BASH_SOURCE} only supports Debian stretch.
     echo Please change your operating system to use this script.
     exit 1
 fi
 
+test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/service-account/service-account.json"
+test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
+test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
+test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
+test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
+test -z ${JAR_VERSION_SUFFIX} && JAR_VERSION_SUFFIX="-SNAPSHOT"
+
 echo "
 This script will run end-to-end tests for Feast Core and Batch Serving.
 
 1. Install gcloud SDK
 2. Install Redis as the job store for Feast Batch Serving.
 4. Install Postgres for persisting Feast metadata.
 5. Install Kafka and Zookeeper as the Source in Feast.
-6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from 
+6. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from
    tests/e2e via pytest.
 "
 
+apt-get -qq update
+apt-get -y install wget netcat kafkacat
+
 
 echo "
 ============================================================
@@ -31,8 +41,8 @@ if [[ ! $(command -v gsutil) ]]; then
   . "${CURRENT_DIR}"/install-google-cloud-sdk.sh
 fi
 
-export GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json
-gcloud auth activate-service-account --key-file /etc/service-account/service-account.json
+export GOOGLE_APPLICATION_CREDENTIALS
+gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS}
 
 
 
@@ -41,10 +51,9 @@ echo "
 Installing Redis at localhost:6379
 ============================================================
 "
-apt-get -qq update
 # Allow starting serving in this Maven Docker image. Default set to not allowed.
 echo "exit 0" > /usr/sbin/policy-rc.d
-apt-get -y install redis-server wget > /var/log/redis.install.log
+apt-get -y install redis-server > /var/log/redis.install.log
 redis-server --daemonize yes
 redis-cli ping
 
@@ -73,24 +82,32 @@ Installing Kafka at localhost:9092
 wget -qO- https://www-eu.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz | tar xz
 mv kafka_2.12-2.3.0/ /tmp/kafka
 nohup /tmp/kafka/bin/zookeeper-server-start.sh /tmp/kafka/config/zookeeper.properties &> /var/log/zookeeper.log 2>&1 &
-sleep 10
+sleep 5
 tail -n10 /var/log/zookeeper.log
 nohup /tmp/kafka/bin/kafka-server-start.sh /tmp/kafka/config/server.properties &> /var/log/kafka.log 2>&1 &
-sleep 30
+sleep 20
 tail -n10 /var/log/kafka.log
-
-echo "
-============================================================
-Building jars for Feast
-============================================================
-"
-
-.prow/scripts/download-maven-cache.sh \
-    --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
-    --output-dir /root/
-
-# Build jars for Feast
-mvn --quiet --batch-mode --define skipTests=true clean package
+kafkacat -b localhost:9092 -L
+
+if [[ ${SKIP_BUILD_JARS} != "true" ]]; then
+  echo "
+  ============================================================
+  Building jars for Feast
+  ============================================================
+  "
+
+  .prow/scripts/download-maven-cache.sh \
+      --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+      --output-dir /root/
+
+  # Build jars for Feast
+  mvn --quiet --batch-mode --define skipTests=true clean package
+
+  ls -lh core/target/*jar
+  ls -lh serving/target/*jar
+else
+  echo "[DEBUG] Skipping building jars"
+fi
 
 echo "
 ============================================================
@@ -142,11 +159,13 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 35
 tail -n10 /var/log/feast-core.log
+nc -w2 localhost 6565 < /dev/null
+
 echo "
 ============================================================
 Starting Feast Warehouse Serving
@@ -155,18 +174,18 @@ Starting Feast Warehouse Serving
 
 DATASET_NAME=feast_$(date +%s)
 
-bq --location=US --project_id=kf-feast mk \
+bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
   --dataset \
   --default_table_expiration 86400 \
-  kf-feast:$DATASET_NAME
+  ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
 
 # Start Feast Online Serving in background
 cat <<EOF > /tmp/serving.store.bigquery.yml
 name: warehouse
 type: BIGQUERY
 bigquery_config:
-  projectId: kf-feast
-  datasetId: $DATASET_NAME
+  projectId: ${GOOGLE_CLOUD_PROJECT}
+  datasetId: ${DATASET_NAME}
 subscriptions:
   - name: "*"
     version: "*"
@@ -183,26 +202,29 @@ feast:
   store:
     config-path: /tmp/serving.store.bigquery.yml
   jobs:
-    staging-location: gs://feast-templocation-kf-feast/staging-location
+    staging-location: ${JOBS_STAGING_LOCATION}
     store-type: REDIS
     bigquery-initial-retry-delay-secs: 1
     bigquery-total-timeout-secs: 900
     store-options:
-      host: $REMOTE_HOST
+      host: localhost
       port: 6379
 grpc:
   port: 6566
   enable-reflection: true
+
 spring:
   main:
     web-environment: false
+
 EOF
 
-nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \
   --spring.config.location=file:///tmp/serving.warehouse.application.yml \
   &> /var/log/feast-serving-warehouse.log &
 sleep 15
 tail -n100 /var/log/feast-serving-warehouse.log
+nc -w2 localhost 6566 < /dev/null
 
 echo "
 ============================================================
@@ -232,16 +254,13 @@ ORIGINAL_DIR=$(pwd)
 cd tests/e2e
 
 set +e
-pytest bq-batch-retrieval.py --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+pytest bq-batch-retrieval.py --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
 TEST_EXIT_CODE=$?
 
 if [[ ${TEST_EXIT_CODE} != 0 ]]; then
   echo "[DEBUG] Printing logs"
   ls -ltrh /var/log/feast*
   cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log
-
-  echo "[DEBUG] Printing Python packages list"
-  pip list
 fi
 
 cd ${ORIGINAL_DIR}
@@ -253,4 +272,4 @@ Cleaning up
 ============================================================
 "
 
-bq rm -r -f kf-feast:$DATASET_NAME
+bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
diff --git a/.prow/scripts/test-end-to-end.sh b/.prow/scripts/test-end-to-end.sh
@@ -4,18 +4,25 @@ set -e
 set -o pipefail
 
 if ! cat /etc/*release | grep -q stretch; then
-    echo ${BASH_SOURCE} only supports Debian stretch. 
+    echo ${BASH_SOURCE} only supports Debian stretch.
     echo Please change your operating system to use this script.
     exit 1
 fi
 
+test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/etc/service-account/service-account.json"
+test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
+test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
+test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
+test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
+test -z ${JAR_VERSION_SUFFIX} && JAR_VERSION_SUFFIX="-SNAPSHOT"
+
 echo "
 This script will run end-to-end tests for Feast Core and Online Serving.
 
 1. Install Redis as the store for Feast Online Serving.
 2. Install Postgres for persisting Feast metadata.
 3. Install Kafka and Zookeeper as the Source in Feast.
-4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from 
+4. Install Python 3.7.4, Feast Python SDK and run end-to-end tests from
    tests/e2e via pytest.
 "
 
@@ -27,7 +34,6 @@ echo "
 Installing Redis at localhost:6379
 ============================================================
 "
-
 # Allow starting serving in this Maven Docker image. Default set to not allowed.
 echo "exit 0" > /usr/sbin/policy-rc.d
 apt-get -y install redis-server > /var/log/redis.install.log
@@ -67,24 +73,24 @@ tail -n10 /var/log/kafka.log
 kafkacat -b localhost:9092 -L
 
 if [[ ${SKIP_BUILD_JARS} != "true" ]]; then
-echo "
-============================================================
-Building jars for Feast
-============================================================
-"
+  echo "
+  ============================================================
+  Building jars for Feast
+  ============================================================
+  "
 
-.prow/scripts/download-maven-cache.sh \
-    --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
-    --output-dir /root/
+  .prow/scripts/download-maven-cache.sh \
+      --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+      --output-dir /root/
 
-# Build jars for Feast
-mvn --quiet --batch-mode --define skipTests=true clean package
+  # Build jars for Feast
+  mvn --quiet --batch-mode --define skipTests=true clean package
 
-ls -lh core/target/*jar
-ls -lh serving/target/*jar
-else
-  echo "[DEBUG] Skipping building jars"
-fi
+  ls -lh core/target/*jar
+  ls -lh serving/target/*jar
+  else
+    echo "[DEBUG] Skipping building jars"
+  fi
 
 echo "
 ============================================================
@@ -122,7 +128,6 @@ spring:
       event.merge.entity_copy_observer: allow
     hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl
     hibernate.ddl-auto: update
-
   datasource:
     url: jdbc:postgresql://localhost:5432/postgres
     username: postgres
@@ -137,7 +142,7 @@ management:
         enabled: false
 EOF
 
-nohup java -jar core/target/feast-core-*-SNAPSHOT.jar \
+nohup java -jar core/target/feast-core-*${JAR_VERSION_SUFFIX}.jar \
   --spring.config.location=file:///tmp/core.application.yml \
   &> /var/log/feast-core.log &
 sleep 35
@@ -167,17 +172,14 @@ feast:
   version: 0.3
   core-host: localhost
   core-grpc-port: 6565
-
   tracing:
     enabled: false
-
   store:
     config-path: /tmp/serving.store.redis.yml
     redis-pool-max-size: 128
     redis-pool-max-idle: 16
-
   jobs:
-    staging-location: gs://feast-templocation-kf-feast/staging-location
+    staging-location: ${JOBS_STAGING_LOCATION}
     store-type:
     store-options: {}
 
@@ -191,11 +193,11 @@ spring:
 
 EOF
 
-nohup java -jar serving/target/feast-serving-*-SNAPSHOT.jar \
+nohup java -jar serving/target/feast-serving-*${JAR_VERSION_SUFFIX}.jar \
   --spring.config.location=file:///tmp/serving.online.application.yml \
   &> /var/log/feast-serving-online.log &
 sleep 15
-tail -n10 /var/log/feast-serving-online.log
+tail -n100 /var/log/feast-serving-online.log
 nc -w2 localhost 6566 < /dev/null
 
 echo "
@@ -233,9 +235,6 @@ if [[ ${TEST_EXIT_CODE} != 0 ]]; then
   echo "[DEBUG] Printing logs"
   ls -ltrh /var/log/feast*
   cat /var/log/feast-serving-online.log /var/log/feast-core.log
-
-  echo "[DEBUG] Printing Python packages list"
-  pip list
 fi
 
 cd ${ORIGINAL_DIR}