Update release workflow to include new docker images (#2108)

* Remove feature transformation server building workflow and merge release workflows. Add/update docker images for lambda feature server, FTS, and java feature server Signed-off-by: Danny Chiao <danny@tecton.ai> * update for ecr Signed-off-by: Danny Chiao <danny@tecton.ai> * Update makefile Signed-off-by: Danny Chiao <danny@tecton.ai> * Separate GCR docker builds into master_only workflow Signed-off-by: Danny Chiao <danny@tecton.ai> * Remove continuous push on merge Signed-off-by: Danny Chiao <danny@tecton.ai> * Update feature server lambda image in constants.py Signed-off-by: Danny Chiao <danny@tecton.ai> * Update docs for lambda Signed-off-by: Danny Chiao <danny@tecton.ai>
feast-dev · Dec 8, 2021 · 5ab6232 · 5ab6232
1 parent 96393dd
commit 5ab6232
Show file tree

Hide file tree

Showing 7 changed files with 119 additions and 254 deletions.
diff --git a/.github/workflows/build_feature_transformation_server.yml b/.github/workflows/build_feature_transformation_server.yml
diff --git a/.github/workflows/java_release.yml b/.github/workflows/java_release.yml
diff --git a/.github/workflows/integration_tests.yml → .github/workflows/master_only.yml b/.github/workflows/integration_tests.yml → .github/workflows/master_only.yml
@@ -1,12 +1,12 @@
-name: integration-tests
+name: master_only
 
 on:
   push:
     branches:
       - master
 
 jobs:
-  build-docker-image:
+  build-lambda-docker-image:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -46,7 +46,7 @@ jobs:
     outputs:
       DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
-    needs: build-docker-image
+    needs: build-lambda-docker-image
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -113,7 +113,7 @@ jobs:
         run: make install-python-ci-dependencies
       - name: Test python
         env:
-          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
+          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
         run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
@@ -128,9 +128,46 @@ jobs:
           verbose: true
       - name: Benchmark python
         env:
-          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
+          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
         run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
       - name: Upload Benchmark Artifact to S3
         run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmarks
+
+  build-all-docker-images:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        component: [ feature-server-aws, feature-server-java, feature-transformation-server ]
+    env:
+      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
+      REGISTRY: gcr.io/kf-feast
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@master
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+          export_default_credentials: true
+      - name: Use gcloud CLI
+        run: gcloud info
+      - run: gcloud auth configure-docker --quiet
+      - name: Get m2 cache
+        run: |
+          infra/scripts/download-maven-cache.sh \
+          --archive-uri ${MAVEN_CACHE} \
+          --output-dir .
+      - name: Build image
+        run: |
+          make build-${{ matrix.component }} REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -13,52 +13,53 @@ jobs:
       version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
       highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Get release version
-      id: get_release_version
-      run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
-    - name: Get release version without prefix
-      id: get_release_version_without_prefix
-      env:
-        RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-      run: |
-        echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
-    - name: Get highest semver
-      id: get_highest_semver
-      env:
-        RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-      run: |
-        source infra/scripts/setup-common-functions.sh
-        SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
-        if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
-          echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
-        fi
-    - name: Check output
-      env:
-        RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
-        VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
-        HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
-      run: |
-        echo $RELEASE_VERSION
-        echo $VERSION_WITHOUT_PREFIX
-        echo $HIGHEST_SEMVER_TAG
+      - uses: actions/checkout@v2
+      - name: Get release version
+        id: get_release_version
+        run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
+      - name: Get release version without prefix
+        id: get_release_version_without_prefix
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+        run: |
+          echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
+      - name: Get highest semver
+        id: get_highest_semver
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+        run: |
+          source infra/scripts/setup-common-functions.sh
+          SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
+          if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
+            echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
+          fi
+      - name: Check output
+        env:
+          RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
+          VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
+          HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
+        run: |
+          echo $RELEASE_VERSION
+          echo $VERSION_WITHOUT_PREFIX
+          echo $HIGHEST_SEMVER_TAG
 
   build-publish-docker-images:
-    runs-on: [ubuntu-latest]
+    runs-on: ubuntu-latest
     needs: get-version
     strategy:
       matrix:
-        component: [jupyter]
+        component: [feature-server-aws, feature-server-java, feature-transformation-server]
     env:
       MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
+      REGISTRY: feastdev
     steps:
       - uses: actions/checkout@v2
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -76,27 +77,26 @@ jobs:
           infra/scripts/download-maven-cache.sh \
           --archive-uri ${MAVEN_CACHE} \
           --output-dir .
-      - name: Build and push versioned images
+      - name: Build image
+        run: |
+          make build-${{ matrix.component }} REGISTRY=${REGISTRY} VERSION=${RELEASE_VERSION}
+        env:
+          RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
+          VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
+          HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
+      - name: Push versioned images
         env:
           RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
           VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
           HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
         run: |
-          docker build --build-arg VERSION=$RELEASE_VERSION \
-            -t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \
-            -t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
-            -t feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
-            -f infra/docker/${{ matrix.component }}/Dockerfile .
-          docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
-          docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
-          
+          make push-${{ matrix.component }} REGISTRY=${REGISTRY} VERSION=${RELEASE_VERSION}
+
           echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
           if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
           then
-            docker tag feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest
-            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
-            docker push feastdev/feast-${{ matrix.component }}:latest
-            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
+            docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest
+            docker push feastdev/${{ matrix.component }}:latest
           fi
 
   publish-helm-charts:
@@ -142,4 +142,7 @@ jobs:
           cd sdk/python
           python3 -m pip install --user --upgrade setuptools wheel twine
           python3 setup.py sdist bdist_wheel
-          python3 -m twine upload --verbose dist/*
+          python3 -m twine upload --verbose dist/*
+
+    # TODO(adchia): publish java sdk once maven repo is updated
+    # See https://github.com/feast-dev/feast-java/blob/master/.github/workflows/release.yml#L104