-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Moving Feast Java back into main repo under java/ package (#1997)
* Moving Feast Java back into main repo under java/ package Signed-off-by: Danny Chiao <danny@tecton.ai> * Update infra scripts with new pom.xml structure Signed-off-by: Danny Chiao <danny@tecton.ai> * Fix test-java-sdk.sh Signed-off-by: Danny Chiao <danny@tecton.ai> * Remove unneeded mirror Signed-off-by: Danny Chiao <danny@tecton.ai> * Update workflows with TODOs Signed-off-by: Danny Chiao <danny@tecton.ai>
- Loading branch information
Showing
287 changed files
with
27,701 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,9 @@ on: | |
push: | ||
branches: | ||
- master | ||
paths: | ||
- "protos/**" | ||
- "sdk/python**" | ||
|
||
jobs: | ||
build-docker-image: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,9 @@ on: | |
push: | ||
branches: | ||
- master | ||
paths: | ||
- "protos/**" | ||
- "sdk/python**" | ||
|
||
jobs: | ||
build-docker-image: | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
name: java_complete | ||
|
||
on: [push, pull_request] | ||
|
||
jobs: | ||
lint-java: | ||
container: gcr.io/kf-feast/feast-ci:latest | ||
runs-on: [ubuntu-latest] | ||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
submodules: 'true' | ||
- name: Lint java | ||
run: make lint-java | ||
|
||
unit-test-java: | ||
runs-on: ubuntu-latest | ||
needs: lint-java | ||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
submodules: 'true' | ||
- name: Set up JDK 11 | ||
uses: actions/setup-java@v1 | ||
with: | ||
java-version: '11' | ||
java-package: jdk | ||
architecture: x64 | ||
- uses: actions/cache@v2 | ||
with: | ||
path: ~/.m2/repository | ||
key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} | ||
restore-keys: | | ||
${{ runner.os }}-ut-maven- | ||
- name: Test java | ||
run: make test-java-with-coverage | ||
- uses: actions/upload-artifact@v2 | ||
with: | ||
name: java-coverage-report | ||
path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ | ||
|
||
integration-test: | ||
runs-on: ubuntu-latest | ||
needs: unit-test-java | ||
services: | ||
redis: | ||
image: redis | ||
ports: | ||
- 6389:6379 | ||
options: >- | ||
--health-cmd "redis-cli ping" | ||
--health-interval 10s | ||
--health-timeout 5s | ||
--health-retries 5 | ||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
submodules: 'true' | ||
- name: Set up JDK 11 | ||
uses: actions/setup-java@v1 | ||
with: | ||
java-version: '11' | ||
java-package: jdk | ||
architecture: x64 | ||
- uses: actions/setup-python@v2 | ||
with: | ||
python-version: '3.7' | ||
architecture: 'x64' | ||
- uses: actions/cache@v2 | ||
with: | ||
path: ~/.m2/repository | ||
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} | ||
restore-keys: | | ||
${{ runner.os }}-it-maven- | ||
- name: Run integration tests | ||
run: make test-java-integration | ||
- name: Save report | ||
uses: actions/upload-artifact@v2 | ||
if: failure() | ||
with: | ||
name: it-report | ||
path: spark/ingestion/target/test-reports/TestSuite.txt | ||
retention-days: 5 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
name: master only | ||
|
||
on: | ||
push: | ||
branches: | ||
- master | ||
tags: | ||
- 'v*.*.*' | ||
|
||
jobs: | ||
build-docker-images: | ||
runs-on: ubuntu-latest | ||
strategy: | ||
matrix: | ||
component: [core, serving] | ||
env: | ||
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar | ||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
submodules: 'true' | ||
- uses: google-github-actions/setup-gcloud@master | ||
with: | ||
version: '290.0.1' | ||
export_default_credentials: true | ||
project_id: ${{ secrets.GCP_PROJECT_ID }} | ||
service_account_key: ${{ secrets.GCP_SA_KEY }} | ||
- run: gcloud auth configure-docker --quiet | ||
- name: Get m2 cache | ||
run: | | ||
infra/scripts/download-maven-cache.sh \ | ||
--archive-uri ${MAVEN_CACHE} \ | ||
--output-dir . | ||
- name: Get version | ||
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV | ||
- name: Build image | ||
run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} | ||
- name: Push image | ||
run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} | ||
- name: Push development Docker image | ||
run: | | ||
if [ ${GITHUB_REF#refs/*/} == "master" ]; then | ||
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop | ||
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,161 @@ | ||
name: release | ||
# TODO(adchia): update DockerHub + Maven repo paths so version can match existing Feast versions. | ||
on: | ||
push: | ||
tags: | ||
- 'v*.*.*' | ||
|
||
jobs: | ||
get-version: | ||
runs-on: ubuntu-latest | ||
outputs: | ||
release_version: ${{ steps.get_release_version.outputs.release_version }} | ||
version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} | ||
highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} | ||
steps: | ||
- uses: actions/checkout@v2 | ||
- name: Get release version | ||
id: get_release_version | ||
run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} | ||
- name: Get release version without prefix | ||
id: get_release_version_without_prefix | ||
env: | ||
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} | ||
run: | | ||
echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} | ||
- name: Get highest semver | ||
id: get_highest_semver | ||
env: | ||
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} | ||
run: | | ||
source infra/scripts/setup-common-functions.sh | ||
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' | ||
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then | ||
echo ::set-output name=highest_semver_tag::$(get_tag_release -m) | ||
fi | ||
- name: Check output | ||
env: | ||
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} | ||
VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} | ||
HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} | ||
run: | | ||
echo $RELEASE_VERSION | ||
echo $VERSION_WITHOUT_PREFIX | ||
echo $HIGHEST_SEMVER_TAG | ||
build-publish-docker-images: | ||
runs-on: [ubuntu-latest] | ||
needs: get-version | ||
strategy: | ||
matrix: | ||
component: [core, serving] | ||
env: | ||
MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar | ||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
submodules: 'true' | ||
- name: Set up QEMU | ||
uses: docker/setup-qemu-action@v1 | ||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@v1 | ||
- name: Login to DockerHub | ||
uses: docker/login-action@v1 | ||
with: | ||
username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master | ||
with: | ||
version: '290.0.1' | ||
export_default_credentials: true | ||
project_id: ${{ secrets.GCP_PROJECT_ID }} | ||
service_account_key: ${{ secrets.GCP_SA_KEY }} | ||
- run: gcloud auth configure-docker --quiet | ||
- name: Get m2 cache | ||
run: | | ||
infra/scripts/download-maven-cache.sh \ | ||
--archive-uri ${MAVEN_CACHE} \ | ||
--output-dir . | ||
- name: Build and push versioned images | ||
env: | ||
RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} | ||
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} | ||
HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} | ||
run: | | ||
docker build --build-arg VERSION=$RELEASE_VERSION \ | ||
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \ | ||
-t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \ | ||
-t feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \ | ||
-f infra/docker/${{ matrix.component }}/Dockerfile . | ||
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} | ||
docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} | ||
echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" | ||
if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] | ||
then | ||
docker tag feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest | ||
docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} gcr.io/kf-feast/feast-${{ matrix.component }}:latest | ||
docker push feastdev/feast-${{ matrix.component }}:latest | ||
docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest | ||
fi | ||
publish-java-sdk: | ||
container: maven:3.6-jdk-11 | ||
runs-on: [ubuntu-latest] | ||
needs: get-version | ||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
submodules: 'true' | ||
- name: Set up JDK 11 | ||
uses: actions/setup-java@v1 | ||
with: | ||
java-version: '11' | ||
java-package: jdk | ||
architecture: x64 | ||
- uses: actions/setup-python@v2 | ||
with: | ||
python-version: '3.6' | ||
architecture: 'x64' | ||
- uses: actions/cache@v2 | ||
with: | ||
path: ~/.m2/repository | ||
key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} | ||
restore-keys: | | ||
${{ runner.os }}-it-maven- | ||
- name: Publish java sdk | ||
env: | ||
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} | ||
GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} | ||
GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} | ||
MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} | ||
run: | | ||
echo -n "$GPG_PUBLIC_KEY" > /root/public-key | ||
echo -n "$GPG_PRIVATE_KEY" > /root/private-key | ||
mkdir -p /root/.m2/ | ||
echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml | ||
infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root | ||
publish-helm-charts: | ||
runs-on: ubuntu-latest | ||
needs: get-version | ||
env: | ||
HELM_VERSION: v2.17.0 | ||
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} | ||
steps: | ||
- uses: actions/checkout@v2 | ||
- uses: GoogleCloudPlatform/github-actions/setup-gcloud@master | ||
with: | ||
version: '290.0.1' | ||
export_default_credentials: true | ||
project_id: ${{ secrets.GCP_PROJECT_ID }} | ||
service_account_key: ${{ secrets.GCP_SA_KEY }} | ||
- run: gcloud auth configure-docker --quiet | ||
- name: Remove previous Helm | ||
run: sudo rm -rf $(which helm) | ||
- name: Install Helm | ||
run: ./infra/scripts/install-helm.sh | ||
- name: Validate all version consistency | ||
run: ./infra/scripts/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX | ||
- name: Publish Helm charts | ||
run: ./infra/scripts/push-helm-charts.sh $VERSION_WITHOUT_PREFIX |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.