Ensure consistency of github workflow (#1157)

* Ensure consistency of github workflow Signed-off-by: Terence <terencelimxp@gmail.com> * Shift GCR latest tag workflow to release Signed-off-by: Terence <terencelimxp@gmail.com> * Fix production build arg Signed-off-by: Terence <terencelimxp@gmail.com> * Fix docker-compose test Signed-off-by: Terence <terencelimxp@gmail.com> * Use single step for dockerhub build and push Signed-off-by: Terence <terencelimxp@gmail.com> * Revert docker compose fix Signed-off-by: Terence <terencelimxp@gmail.com> * Simplify to single build push stage Signed-off-by: Terence <terencelimxp@gmail.com> * Remove build and just tag Signed-off-by: Terence <terencelimxp@gmail.com>
feast-dev · Nov 24, 2020 · b60a3f7 · b60a3f7
1 parent 89871f1
commit b60a3f7
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 59 deletions.
diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml
@@ -25,7 +25,7 @@ jobs:
           --archive-uri ${MAVEN_CACHE} \
           --output-dir .
       - name: Build image
-        run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} REVISION=dev
+        run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
       - name: Push image
         run: |
           docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA}

diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -30,38 +30,15 @@ jobs:
       - name: Get version
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Build image
-        run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} REVISION=${RELEASE_VERSION}
+        run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
       - name: Push image
-        run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} REVISION=${RELEASE_VERSION}
+        run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
       - name: Push development Docker image
         run: |
           if [ ${GITHUB_REF#refs/*/} == "master" ]; then
             docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop
             docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop
           fi
-      - name: Push versioned Docker image
-        run: |
-          source infra/scripts/setup-common-functions.sh
-          # Build and push semver tagged commits
-          # Regular expression should match MAJOR.MINOR.PATCH[-PRERELEASE[.IDENTIFIER]]
-          # eg. v0.7.1 v0.7.2-alpha v0.7.2-rc.1
-          SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
-          if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
-            VERSION_WITHOUT_PREFIX=${RELEASE_VERSION:1}
-
-            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
-            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
-
-            # Also update "latest" image if tagged commit is pushed to stable branch
-            HIGHEST_SEMVER_TAG=$(get_tag_release -m)
-            echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
-
-            if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
-            then
-              docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
-              docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
-            fi
-          fi
 
   publish-ingestion-jar:
     runs-on: [ self-hosted ]

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -69,36 +69,32 @@ jobs:
           export_default_credentials: true
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           service_account_key: ${{ secrets.GCP_SA_KEY }}
+      - run: gcloud auth configure-docker --quiet
       - name: Get m2 cache
         run: |
           infra/scripts/download-maven-cache.sh \
           --archive-uri ${MAVEN_CACHE} \
           --output-dir .
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        env:
-          RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
-        with:
-          push: true
-          context: .
-          file: ./infra/docker/${{ matrix.component }}/Dockerfile
-          tags: feastdev/feast-${{ matrix.component }}:${{ needs.get-version.outputs.release_version }}
-          build-args: |
-            REVISION=$RELEASE_VERSION
-      - name: Build and push latest
-        uses: docker/build-push-action@v2
+      - name: Build and push versioned images
         env:
           RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
           VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
           HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
-        with:
-          if: ${VERSION_WITHOUT_PREFIX} == ${HIGHEST_SEMVER_TAG:1}
-          push: true
-          context: .
-          file: ./infra/docker/${{ matrix.component }}/Dockerfile
-          tags: feastdev/feast-${{ matrix.component }}:latest
-          build-args: |
-            REVISION=$RELEASE_VERSION
+        run: |
+          docker build --build-arg VERSION=$RELEASE_VERSION \
+            -t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \
+            -t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
+            -f infra/docker/${{ matrix.component }}/Dockerfile .
+          docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
+          
+          echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
+          if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
+          then
+            docker tag feastdev/feast-${{ matrix.component }}:latest gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
+            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:latest gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
+            docker push feastdev/feast-${{ matrix.component }}:latest
+            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
+          fi
 
   publish-helm-charts:
     runs-on:  ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -142,13 +142,13 @@ push-jupyter-docker:
 	docker push $(REGISTRY)/feast-jupyter:$(VERSION)
 
 build-core-docker:
-	docker build $(if, $(REVISION),--build-arg REVISION=$(REVISION),) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
+	docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
 
 build-jobservice-docker:
 	docker build -t $(REGISTRY)/feast-jobservice:$(VERSION) -f infra/docker/jobservice/Dockerfile .
 
 build-serving-docker:
-	docker build $(if, $(REVISION),--build-arg REVISION=$(REVISION),) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
+	docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
 
 build-ci-docker:
 	docker build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile .

diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile
@@ -29,8 +29,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true
 
 COPY . .
 
-ARG REVISION=dev
-RUN mvn --also-make --projects core -Drevision=$REVISION \
+ARG VERSION=dev
+RUN mvn --also-make --projects core -Drevision=$VERSION \
   -DskipUTs=true --batch-mode clean package
 
 #
@@ -46,9 +46,9 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa
 # ============================================================
 
 FROM openjdk:11-jre as production
-ARG REVISION=dev
+ARG VERSION=dev
 
-COPY --from=builder /build/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar
+COPY --from=builder /build/core/target/feast-core-$VERSION-exec.jar /opt/feast/feast-core.jar
 COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe
 
 CMD ["java",\

diff --git a/infra/docker/jobservice/Dockerfile b/infra/docker/jobservice/Dockerfile
@@ -1,7 +1,6 @@
 FROM jupyter/pyspark-notebook:ae5f7e104dd5
 
 USER root
-ARG REVISION
 WORKDIR /feast
 
 COPY sdk/python sdk/python

diff --git a/infra/docker/jupyter/Dockerfile b/infra/docker/jupyter/Dockerfile
@@ -1,7 +1,6 @@
 FROM jupyter/pyspark-notebook:ae5f7e104dd5
 
 USER root
-ARG REVISION
 WORKDIR /feast
 
 COPY sdk/python sdk/python

diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile
@@ -29,8 +29,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true
 
 COPY . .
 
-ARG REVISION=dev
-RUN mvn --also-make --projects serving -Drevision=$REVISION \
+ARG VERSION=dev
+RUN mvn --also-make --projects serving -Drevision=$VERSION \
   -DskipUTs=true --batch-mode clean package
 #
 # Download grpc_health_probe to run health check for Feast Serving
@@ -45,8 +45,8 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa
 # ============================================================
 
 FROM openjdk:11-jre-slim as production
-ARG REVISION=dev
-COPY --from=builder /build/serving/target/feast-serving-$REVISION-exec.jar /opt/feast/feast-serving.jar
+ARG VERSION=dev
+COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar
 COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe
 CMD ["java",\
      "-Xms1024m",\