fix dataflow e2e

feast-dev · Jul 14, 2020 · d8627e6 · d8627e6
1 parent 4e80613
commit d8627e6
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 7 deletions.
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -274,7 +274,7 @@ presubmits:
 
   - name: test-end-to-end-batch-dataflow
     decorate: true
-    always_run: false
+    always_run: true
     spec:
       volumes:
       - name: service-account-df
@@ -311,7 +311,7 @@ presubmits:
         - -c
         - |
           infra/scripts/download-maven-cache.sh \
-            --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+            --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \
             --output-dir $PWD/
 
           infra/scripts/publish-docker-image.sh \
@@ -440,7 +440,7 @@ postsubmits:
         - -c
         - |
           infra/scripts/download-maven-cache.sh \
-            --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
+            --archive-uri gs://feast-templocation-kf-feast/.m2.2020-07-13.tar \
             --output-dir $PWD/
 
           if [ $PULL_BASE_REF == "master" ]; then

diff --git a/core/pom.xml b/core/pom.xml
@@ -209,12 +209,12 @@
         <dependency>
             <groupId>com.google.api-client</groupId>
             <artifactId>google-api-client</artifactId>
-            <version>1.27.0</version>
+            <version>1.30.9</version>
         </dependency>
         <dependency>
             <groupId>com.google.apis</groupId>
             <artifactId>google-api-services-dataflow</artifactId>
-            <version>v1b3-rev266-1.25.0</version>
+            <version>v1b3-rev20200305-1.30.9</version>
         </dependency>
         <dependency>
             <groupId>org.hibernate</groupId>

diff --git a/infra/scripts/test-end-to-end-batch-dataflow.sh b/infra/scripts/test-end-to-end-batch-dataflow.sh
@@ -203,7 +203,7 @@ Export required environment variables
 ============================================================
 "
 
-export TEMP_BUCKET=$TEMP_BUCKET
+export TEMP_BUCKET=$TEMP_BUCKET/$HELM_RELEASE_NAME/$(date +%s)
 export DATASET_NAME=$DATASET_NAME
 export GCLOUD_PROJECT=$GCLOUD_PROJECT
 export GCLOUD_NETWORK=$GCLOUD_NETWORK

diff --git a/ingestion/pom.xml b/ingestion/pom.xml
@@ -70,6 +70,16 @@
                   <shadedPattern>com.google.cloud.bigquery.vendor</shadedPattern>
                 </relocation>
               </relocations>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
             </configuration>
           </execution>
         </executions>

diff --git a/tests/e2e/bq/bq-batch-retrieval.py b/tests/e2e/bq/bq-batch-retrieval.py
@@ -243,7 +243,7 @@ def test_batch_get_historical_features_with_gs_path(client, gcs_path):
     uri = urlparse(gcs_path)
     bucket = uri.hostname
     ts = int(time.time())
-    remote_path = str(uri.path).strip("/") + f"{ts}/{file_name}"
+    remote_path = str(uri.path).lstrip("/") + f"{ts}/{file_name}"
 
     # Upload file to gcs
     storage_client = storage.Client(project=None)