diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index 6c3bb2d087..8eec3b27c9 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -142,3 +142,11 @@ jobs: with: name: load-test-results path: load-test-output/ + + tests-docker-compose: + needs: build-push-docker-images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Test docker compose + run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA} diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 9df9094b10..a5244fa22b 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -59,10 +59,3 @@ jobs: docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest fi fi - - tests-docker-compose: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Test docker compose - run: ./infra/scripts/test-docker-compose.sh diff --git a/infra/docker-compose/jobcontroller/jobcontroller.yml b/infra/docker-compose/jobcontroller/jobcontroller.yml index 4d28d62f8c..a8be1f7b6c 100644 --- a/infra/docker-compose/jobcontroller/jobcontroller.yml +++ b/infra/docker-compose/jobcontroller/jobcontroller.yml @@ -1,7 +1,7 @@ feast: core-host: core jobs: - polling_interval_milliseconds: 20000 + polling_interval_milliseconds: 10000 job_update_timeout_seconds: 240 active_runner: direct runners: diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 0d6e99f860..35e1593dd6 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -7,6 +7,7 @@ echo " Running Docker Compose tests with pytest at 'tests/e2e' ============================================================ " +LATEST_GH_COMMIT_SHA=$1 clean_up () { ARG=$? @@ -26,6 +27,10 @@ export COMPOSE_INTERACTIVE_NO_CLI=1 cd ${PROJECT_ROOT_DIR}/infra/docker-compose/ cp .env.sample .env +# Replace FEAST_VERSION with latest github image SHA +export FEAST_VERSION=$LATEST_GH_COMMIT_SHA +echo "Testing docker-compose setup with version SHA, $FEAST_VERSION." + # Start Docker Compose containers docker-compose up -d @@ -45,6 +50,12 @@ export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSett # Wait for Feast Core to be ready ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120 +# Get Feast Job Controller container IP address +export FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jobcontroller_1) + +# Wait for Feast Job Controller to be ready +"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_JOB_CONTROLLER_CONTAINER_IP_ADDRESS}:6570 --timeout=120 + # Get Feast Online Serving container IP address export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online_serving_1) @@ -52,4 +63,4 @@ export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .N ${PROJECT_ROOT_DIR}/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}:6566 --timeout=120 # Run e2e tests for Redis -docker exec feast_jupyter_1 bash -c 'cd /feast/tests/e2e/redis && pytest --verbose -rs basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online_serving:6566 --kafka_brokers=kafka:9092' +docker exec feast_jupyter_1 bash -c 'cd /feast/tests/e2e/redis && pytest --verbose -rs basic-ingest-redis-serving.py --core_url core:6565 --serving_url=online_serving:6566 --jobcontroller_url=jobcontroller:6570 --kafka_brokers=kafka:9092'