Skip to content

Commit

Permalink
Moving Feast Java back into main repo under java/ package (#1997)
Browse files Browse the repository at this point in the history
* Moving Feast Java back into main repo under java/ package

Signed-off-by: Danny Chiao <danny@tecton.ai>

* Update infra scripts with new pom.xml structure

Signed-off-by: Danny Chiao <danny@tecton.ai>

* Fix test-java-sdk.sh

Signed-off-by: Danny Chiao <danny@tecton.ai>

* Remove unneeded mirror

Signed-off-by: Danny Chiao <danny@tecton.ai>

* Update workflows with TODOs

Signed-off-by: Danny Chiao <danny@tecton.ai>
  • Loading branch information
adchia authored Nov 12, 2021
1 parent a9a3a16 commit 63680ba
Show file tree
Hide file tree
Showing 287 changed files with 27,701 additions and 18 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/build_feature_server.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ on:
push:
branches:
- master
paths:
- "protos/**"
- "sdk/python**"

jobs:
build-docker-image:
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/build_feature_transformation_server.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ on:
push:
branches:
- master
paths:
- "protos/**"
- "sdk/python**"

jobs:
build-docker-image:
Expand Down
83 changes: 83 additions & 0 deletions .github/workflows/java_complete.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
name: java_complete

on: [push, pull_request]

jobs:
lint-java:
container: gcr.io/kf-feast/feast-ci:latest
runs-on: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- name: Lint java
run: make lint-java

unit-test-java:
runs-on: ubuntu-latest
needs: lint-java
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- name: Set up JDK 11
uses: actions/setup-java@v1
with:
java-version: '11'
java-package: jdk
architecture: x64
- uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-ut-maven-
- name: Test java
run: make test-java-with-coverage
- uses: actions/upload-artifact@v2
with:
name: java-coverage-report
path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/

integration-test:
runs-on: ubuntu-latest
needs: unit-test-java
services:
redis:
image: redis
ports:
- 6389:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- name: Set up JDK 11
uses: actions/setup-java@v1
with:
java-version: '11'
java-package: jdk
architecture: x64
- uses: actions/setup-python@v2
with:
python-version: '3.7'
architecture: 'x64'
- uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-it-maven-
- name: Run integration tests
run: make test-java-integration
- name: Save report
uses: actions/upload-artifact@v2
if: failure()
with:
name: it-report
path: spark/ingestion/target/test-reports/TestSuite.txt
retention-days: 5
45 changes: 45 additions & 0 deletions .github/workflows/java_master_only.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: master only

on:
push:
branches:
- master
tags:
- 'v*.*.*'

jobs:
build-docker-images:
runs-on: ubuntu-latest
strategy:
matrix:
component: [core, serving]
env:
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- uses: google-github-actions/setup-gcloud@master
with:
version: '290.0.1'
export_default_credentials: true
project_id: ${{ secrets.GCP_PROJECT_ID }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- run: gcloud auth configure-docker --quiet
- name: Get m2 cache
run: |
infra/scripts/download-maven-cache.sh \
--archive-uri ${MAVEN_CACHE} \
--output-dir .
- name: Get version
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Build image
run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
- name: Push image
run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
- name: Push development Docker image
run: |
if [ ${GITHUB_REF#refs/*/} == "master" ]; then
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop
fi
161 changes: 161 additions & 0 deletions .github/workflows/java_release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
name: release
# TODO(adchia): update DockerHub + Maven repo paths so version can match existing Feast versions.
on:
push:
tags:
- 'v*.*.*'

jobs:
get-version:
runs-on: ubuntu-latest
outputs:
release_version: ${{ steps.get_release_version.outputs.release_version }}
version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
steps:
- uses: actions/checkout@v2
- name: Get release version
id: get_release_version
run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
- name: Get release version without prefix
id: get_release_version_without_prefix
env:
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
run: |
echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
- name: Get highest semver
id: get_highest_semver
env:
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
run: |
source infra/scripts/setup-common-functions.sh
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
fi
- name: Check output
env:
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
run: |
echo $RELEASE_VERSION
echo $VERSION_WITHOUT_PREFIX
echo $HIGHEST_SEMVER_TAG
build-publish-docker-images:
runs-on: [ubuntu-latest]
needs: get-version
strategy:
matrix:
component: [core, serving]
env:
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
version: '290.0.1'
export_default_credentials: true
project_id: ${{ secrets.GCP_PROJECT_ID }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- run: gcloud auth configure-docker --quiet
- name: Get m2 cache
run: |
infra/scripts/download-maven-cache.sh \
--archive-uri ${MAVEN_CACHE} \
--output-dir .
- name: Build and push versioned images
env:
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }}
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }}
run: |
docker build --build-arg VERSION=$RELEASE_VERSION \
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
-t feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \
-f infra/docker/${{ matrix.component }}/Dockerfile .
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ]
then
docker tag feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
docker push feastdev/feast-${{ matrix.component }}:latest
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
fi
publish-java-sdk:
container: maven:3.6-jdk-11
runs-on: [ubuntu-latest]
needs: get-version
steps:
- uses: actions/checkout@v2
with:
submodules: 'true'
- name: Set up JDK 11
uses: actions/setup-java@v1
with:
java-version: '11'
java-package: jdk
architecture: x64
- uses: actions/setup-python@v2
with:
python-version: '3.6'
architecture: 'x64'
- uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-it-maven-
- name: Publish java sdk
env:
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }}
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }}
run: |
echo -n "$GPG_PUBLIC_KEY" > /root/public-key
echo -n "$GPG_PRIVATE_KEY" > /root/private-key
mkdir -p /root/.m2/
echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml
infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root
publish-helm-charts:
runs-on: ubuntu-latest
needs: get-version
env:
HELM_VERSION: v2.17.0
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
steps:
- uses: actions/checkout@v2
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
version: '290.0.1'
export_default_credentials: true
project_id: ${{ secrets.GCP_PROJECT_ID }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- run: gcloud auth configure-docker --quiet
- name: Remove previous Helm
run: sudo rm -rf $(which helm)
- name: Install Helm
run: ./infra/scripts/install-helm.sh
- name: Validate all version consistency
run: ./infra/scripts/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX
- name: Publish Helm charts
run: ./infra/scripts/push-helm-charts.sh $VERSION_WITHOUT_PREFIX
8 changes: 4 additions & 4 deletions .prow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ presubmits:
spec:
containers:
- image: maven:3.6-jdk-11
command: ["infra/scripts/test-core-ingestion.sh"]
command: ["infra/scripts/test-java-core-ingestion.sh"]
resources:
requests:
cpu: "2000m"
Expand All @@ -18,7 +18,7 @@ presubmits:
spec:
containers:
- image: maven:3.6-jdk-8
command: ["infra/scripts/test-core-ingestion.sh"]
command: ["infra/scripts/test-java-core-ingestion.sh"]
resources:
requests:
cpu: "2000m"
Expand All @@ -31,7 +31,7 @@ presubmits:
spec:
containers:
- image: maven:3.6-jdk-11
command: ["infra/scripts/test-serving.sh"]
command: ["infra/scripts/test-java-serving.sh"]
skip_branches:
- ^v0\.(3|4)-branch$

Expand All @@ -41,7 +41,7 @@ presubmits:
spec:
containers:
- image: maven:3.6-jdk-8
command: ["infra/scripts/test-serving.sh"]
command: ["infra/scripts/test-java-serving.sh"]
branches:
- ^v0\.(3|4)-branch$

Expand Down
Loading

0 comments on commit 63680ba

Please sign in to comment.