diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 99f565269a..6f803ea42d 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -132,25 +132,32 @@ jobs: - build-push-docker-images - publish-ingestion-jar runs-on: ubuntu-latest + env: + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-develop.jar steps: - uses: actions/checkout@v2 + - name: Download ingestion jar + uses: actions/download-artifact@v2 + with: + name: ingestion-jar + path: ./infra/docker-compose/ - name: Test docker compose run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA} publish-ingestion-jar: - runs-on: [self-hosted] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 + - name: Cache local Maven repository + uses: actions/cache@v2 with: - maven-version: 3.6.3 + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: build-jar env: # Try to add retries to prevent connection resets @@ -158,6 +165,10 @@ jobs: # https://github.com/actions/virtual-environments/issues/1499#issuecomment-718396233 MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false MAVEN_EXTRA_OPTS: -X - run: make build-java-no-tests REVISION=${GITHUB_SHA} - - name: copy to gs - run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/ + run: make build-java-no-tests REVISION=develop + - name: Upload ingestion jar + uses: actions/upload-artifact@v2 + with: + name: ingestion-jar + path: spark/ingestion/target/feast-ingestion-spark-develop.jar + retention-days: 1 diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index f168b48ab7..deaf005889 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -63,7 +63,7 @@ jobs: fi publish-ingestion-jar: - runs-on: [ self-hosted ] + runs-on: ubuntu-latest env: PUBLISH_BUCKET: feast-jobs steps: @@ -72,12 +72,18 @@ jobs: with: version: '290.0.1' export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 + - name: Cache local Maven repository + uses: actions/cache@v2 with: - maven-version: 3.6.3 + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Publish develop version of ingestion job run: | if [ ${GITHUB_REF#refs/*/} == "master" ]; then diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index ea98b441d6..8c48fbc976 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,4 +2,5 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=develop FEAST_CORE_CONFIG=./core/core.yml FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml -GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json \ No newline at end of file +GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json +INGESTION_JAR_PATH=https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-develop.jar \ No newline at end of file diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 90c94e0055..4fea101478 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -36,6 +36,7 @@ services: FEAST_HISTORICAL_FEATURE_OUTPUT_LOCATION: file:///shared/historical_feature_output FEAST_HISTORICAL_FEATURE_OUTPUT_FORMAT: parquet FEAST_REDIS_HOST: redis + FEAST_SPARK_INGESTION_JAR: ${INGESTION_JAR_PATH} jupyter: image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION} diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh index 3dab0513f6..4679cbb9b1 100755 --- a/infra/scripts/test-end-to-end-gcp.sh +++ b/infra/scripts/test-end-to-end-gcp.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash export DISABLE_SERVICE_FIXTURES=1 +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 60f9c33a14..03beb22ab4 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" + +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 441864ba50..eb085a2c01 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -47,7 +47,9 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture(scope="session", params=[True, False]) +@pytest.fixture( + scope="session", params=[False], +) def enable_auth(request): return request.param