Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ensure consistency of github workflow #1157

Merged
merged 8 commits into from
Nov 19, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/complete.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
--archive-uri ${MAVEN_CACHE} \
--output-dir .
- name: Build image
run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} REVISION=dev
run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
- name: Push image
run: |
docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA}
Expand Down
27 changes: 2 additions & 25 deletions .github/workflows/master_only.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,38 +29,15 @@ jobs:
- name: Get version
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Build image
run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} REVISION=${RELEASE_VERSION}
run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
- name: Push image
run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} REVISION=${RELEASE_VERSION}
run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
- name: Push development Docker image
run: |
if [ ${GITHUB_REF#refs/*/} == "master" ]; then
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop
fi
- name: Push versioned Docker image
run: |
source infra/scripts/setup-common-functions.sh
# Build and push semver tagged commits
# Regular expression should match MAJOR.MINOR.PATCH[-PRERELEASE[.IDENTIFIER]]
# eg. v0.7.1 v0.7.2-alpha v0.7.2-rc.1
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
VERSION_WITHOUT_PREFIX=${RELEASE_VERSION:1}

docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}

# Also update "latest" image if tagged commit is pushed to stable branch
HIGHEST_SEMVER_TAG=$(get_tag_release -m)
echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"

if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
then
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
fi
fi

publish-ingestion-jar:
runs-on: ubuntu-latest
Expand Down
38 changes: 17 additions & 21 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,36 +68,32 @@ jobs:
export_default_credentials: true
project_id: ${{ secrets.GCP_PROJECT_ID }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- run: gcloud auth configure-docker --quiet
- name: Get m2 cache
run: |
infra/scripts/download-maven-cache.sh \
--archive-uri ${MAVEN_CACHE} \
--output-dir .
- name: Build and push
uses: docker/build-push-action@v2
env:
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
with:
push: true
context: .
file: ./infra/docker/${{ matrix.component }}/Dockerfile
tags: feastdev/feast-${{ matrix.component }}:${{ needs.get-version.outputs.release_version }}
build-args: |
REVISION=$RELEASE_VERSION
- name: Build and push latest
uses: docker/build-push-action@v2
- name: Build and push versioned images
env:
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
with:
if: ${VERSION_WITHOUT_PREFIX} == ${HIGHEST_SEMVER_TAG:1}
push: true
context: .
file: ./infra/docker/${{ matrix.component }}/Dockerfile
tags: feastdev/feast-${{ matrix.component }}:latest
build-args: |
REVISION=$RELEASE_VERSION
run: |
docker build --build-arg VERSION=$RELEASE_VERSION \
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
-f infra/docker/${{ matrix.component }}/Dockerfile .
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}

echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
then
docker tag feastdev/feast-${{ matrix.component }}:latest gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:latest gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
docker push feastdev/feast-${{ matrix.component }}:latest
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
fi

publish-helm-charts:
runs-on: ubuntu-latest
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -142,13 +142,13 @@ push-jupyter-docker:
docker push $(REGISTRY)/feast-jupyter:$(VERSION)

build-core-docker:
docker build $(if, $(REVISION),--build-arg REVISION=$(REVISION),) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .

build-jobservice-docker:
docker build -t $(REGISTRY)/feast-jobservice:$(VERSION) -f infra/docker/jobservice/Dockerfile .

build-serving-docker:
docker build $(if, $(REVISION),--build-arg REVISION=$(REVISION),) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .
docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile .

build-ci-docker:
docker build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile .
Expand Down
8 changes: 4 additions & 4 deletions infra/docker/core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true

COPY . .

ARG REVISION=dev
RUN mvn --also-make --projects core -Drevision=$REVISION \
ARG VERSION=dev
RUN mvn --also-make --projects core -Drevision=$VERSION \
-DskipUTs=true --batch-mode clean package

#
Expand All @@ -46,9 +46,9 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa
# ============================================================

FROM openjdk:11-jre as production
ARG REVISION=dev
ARG VERSION=dev

COPY --from=builder /build/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar
COPY --from=builder /build/core/target/feast-core-$VERSION-exec.jar /opt/feast/feast-core.jar
COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe

CMD ["java",\
Expand Down
1 change: 0 additions & 1 deletion infra/docker/jobservice/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
FROM jupyter/pyspark-notebook:ae5f7e104dd5

USER root
ARG REVISION
WORKDIR /feast

COPY sdk/python sdk/python
Expand Down
1 change: 0 additions & 1 deletion infra/docker/jupyter/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
FROM jupyter/pyspark-notebook:ae5f7e104dd5

USER root
ARG REVISION
WORKDIR /feast

COPY sdk/python sdk/python
Expand Down
8 changes: 4 additions & 4 deletions infra/docker/serving/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true

COPY . .

ARG REVISION=dev
RUN mvn --also-make --projects serving -Drevision=$REVISION \
ARG VERSION=dev
RUN mvn --also-make --projects serving -Drevision=$VERSION \
-DskipUTs=true --batch-mode clean package
#
# Download grpc_health_probe to run health check for Feast Serving
Expand All @@ -45,8 +45,8 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa
# ============================================================

FROM openjdk:11-jre-slim as production
ARG REVISION=dev
COPY --from=builder /build/serving/target/feast-serving-$REVISION-exec.jar /opt/feast/feast-serving.jar
ARG VERSION=dev
COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar
COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe
CMD ["java",\
"-Xms1024m",\
Expand Down