diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..0d7476a82b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,206 @@ +name: publish + +on: + push: + tags: + - 'v*.*.*' + +jobs: + get-version: + runs-on: ubuntu-latest + outputs: + release_version: ${{ steps.get_release_version.outputs.release_version }} + version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + steps: + - uses: actions/checkout@v2 + - name: Get release version + id: get_release_version + run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} + - name: Get release version without prefix + id: get_release_version_without_prefix + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} + - name: Get highest semver + id: get_highest_semver + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + fi + - name: Check output + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + run: | + echo $RELEASE_VERSION + echo $VERSION_WITHOUT_PREFIX + echo $HIGHEST_SEMVER_TAG + + build-publish-docker-images: + runs-on: ubuntu-latest + needs: get-version + strategy: + matrix: + component: [feature-server-python-aws, feature-server-java, feature-transformation-server] + env: + MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar + REGISTRY: feastdev + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Use gcloud CLI + run: gcloud info + - run: gcloud auth configure-docker --quiet + - name: Get m2 cache + run: | + infra/scripts/download-maven-cache.sh \ + --archive-uri ${MAVEN_CACHE} \ + --output-dir . + - name: Build image + run: | + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + env: + RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} + - name: Push versioned images + env: + RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} + run: | + make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" + if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] + then + docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest + docker push feastdev/${{ matrix.component }}:latest + fi + + publish-helm-charts: + runs-on: ubuntu-latest + needs: get-version + env: + HELM_VERSION: v3.8.0 + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + steps: + - uses: actions/checkout@v2 + - uses: google-github-actions/setup-gcloud@master + with: + version: '290.0.1' + export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - run: gcloud auth configure-docker --quiet + - name: Remove previous Helm + run: sudo rm -rf $(which helm) + - name: Install Helm + run: ./infra/scripts/helm/install-helm.sh + - name: Validate Helm chart prior to publishing + run: ./infra/scripts/helm/validate-helm-chart-publish.sh + - name: Validate all version consistency + run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX + - name: Publish Helm charts + run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX + + publish-python-sdk: + runs-on: ubuntu-latest + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + container: python:3.7 + steps: + - uses: actions/checkout@v2 + - name: Install pip-tools + run: pip install pip-tools + - name: Install dependencies + run: make install-python-ci-dependencies PYTHON=3.7 + - name: Publish Python Package + run: | + cd sdk/python + python3 -m pip install --user --upgrade setuptools wheel twine + python3 setup.py sdist bdist_wheel + python3 -m twine upload --verbose dist/* + + publish-python-sdk-no-telemetry: + runs-on: ubuntu-latest + needs: get-version + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + container: python:3.7 + steps: + - uses: actions/checkout@v2 + - name: Install pip-tools + run: pip install pip-tools + - name: Install dependencies + run: make install-python-ci-dependencies PYTHON=3.7 + - name: Publish Python Package + env: + SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.get-version.outputs.version_without_prefix }} + run: | + cd sdk/python + sed -i 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py + sed -i 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py + python3 -m pip install --user --upgrade setuptools wheel twine + python3 setup.py sdist bdist_wheel + python3 -m twine upload --verbose dist/* + + publish-java-sdk: + container: maven:3.6-jdk-11 + runs-on: ubuntu-latest + needs: get-version + steps: + - uses: actions/checkout@v2 + with: + submodules: 'true' + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: '11' + java-package: jdk + architecture: x64 + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + architecture: 'x64' + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-it-maven- + - name: Publish java sdk + env: + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} + run: | + echo -n "$GPG_PUBLIC_KEY" > /root/public-key + echo -n "$GPG_PRIVATE_KEY" > /root/private-key + mkdir -p /root/.m2/ + echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml + infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0e5419af7..9411e8fc09 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,206 +1,39 @@ name: release on: - push: - tags: - - 'v*.*.*' + workflow_dispatch: + inputs: + dry_run: + description: 'Dry Run' + required: true + default: 'true' + type: boolean + token: + description: 'Personal Access Token' + required: true + default: "" + type: string jobs: - get-version: - runs-on: ubuntu-latest - outputs: - release_version: ${{ steps.get_release_version.outputs.release_version }} - version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - steps: - - uses: actions/checkout@v2 - - name: Get release version - id: get_release_version - run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} - - name: Get release version without prefix - id: get_release_version_without_prefix - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} - - name: Get highest semver - id: get_highest_semver - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) - fi - - name: Check output - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - run: | - echo $RELEASE_VERSION - echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG - - build-publish-docker-images: + release: + name: release runs-on: ubuntu-latest - needs: get-version - strategy: - matrix: - component: [feature-server-python-aws, feature-server-java, feature-transformation-server] env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: feastdev - steps: - - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@master - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Use gcloud CLI - run: gcloud info - - run: gcloud auth configure-docker --quiet - - name: Get m2 cache - run: | - infra/scripts/download-maven-cache.sh \ - --archive-uri ${MAVEN_CACHE} \ - --output-dir . - - name: Build image - run: | - make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - - name: Push versioned images - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - run: | - make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} - - echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" - if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] - then - docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest - docker push feastdev/${{ matrix.component }}:latest - fi - - publish-helm-charts: - runs-on: ubuntu-latest - needs: get-version - env: - HELM_VERSION: v3.8.0 - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - steps: - - uses: actions/checkout@v2 - - uses: google-github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - - run: gcloud auth configure-docker --quiet - - name: Remove previous Helm - run: sudo rm -rf $(which helm) - - name: Install Helm - run: ./infra/scripts/helm/install-helm.sh - - name: Validate Helm chart prior to publishing - run: ./infra/scripts/helm/validate-helm-chart-publish.sh - - name: Validate all version consistency - run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX - - name: Publish Helm charts - run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX - - publish-python-sdk: - runs-on: ubuntu-latest - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - container: python:3.7 - steps: - - uses: actions/checkout@v2 - - name: Install pip-tools - run: pip install pip-tools - - name: Install dependencies - run: make install-python-ci-dependencies PYTHON=3.7 - - name: Publish Python Package - run: | - cd sdk/python - python3 -m pip install --user --upgrade setuptools wheel twine - python3 setup.py sdist bdist_wheel - python3 -m twine upload --verbose dist/* - - publish-python-sdk-no-telemetry: - runs-on: ubuntu-latest - needs: get-version - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - container: python:3.7 - steps: - - uses: actions/checkout@v2 - - name: Install pip-tools - run: pip install pip-tools - - name: Install dependencies - run: make install-python-ci-dependencies PYTHON=3.7 - - name: Publish Python Package - env: - SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.get-version.outputs.version_without_prefix }} - run: | - cd sdk/python - sed -i 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py - sed -i 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py - python3 -m pip install --user --upgrade setuptools wheel twine - python3 setup.py sdist bdist_wheel - python3 -m twine upload --verbose dist/* - - publish-java-sdk: - container: maven:3.6-jdk-11 - runs-on: ubuntu-latest - needs: get-version + GITHUB_TOKEN: ${{ github.event.inputs.token }} steps: - - uses: actions/checkout@v2 - with: - submodules: 'true' - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - java-package: jdk - architecture: x64 - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - architecture: 'x64' - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-it-maven- - - name: Publish java sdk - env: - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} - run: | - echo -n "$GPG_PUBLIC_KEY" > /root/public-key - echo -n "$GPG_PRIVATE_KEY" > /root/private-key - mkdir -p /root/.m2/ - echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml - infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root + - name: Checkout + uses: actions/checkout@v2 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: '16' + - name: Release (Dry Run) + if: github.event.inputs.dry_run == 'true' + run: | + npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run + - name: Release + if: github.event.inputs.dry_run == 'false' + run: | + npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release \ No newline at end of file diff --git a/.releaserc.js b/.releaserc.js new file mode 100644 index 0000000000..a72c333515 --- /dev/null +++ b/.releaserc.js @@ -0,0 +1,53 @@ +// Release script for semantic-release.gitbook.io + +const execSync = require("child_process").execSync; + +// We have to dynamically generate all the supported branches for Feast because we use the `vA.B-branch` pattern for +// maintenance branches +MAJOR_VERSION_MAX=0 +MINOR_VERSION_MIN=15 +MINOR_VERSION_MAX=50 + +possible_branches = [] +possible_branches.push({ + name: "master" +}) + +for (let step_major = 0; step_major <= MAJOR_VERSION_MAX; step_major++) { + for (let step_minor = MINOR_VERSION_MIN; step_minor <= MINOR_VERSION_MAX; step_minor++) { + possible_branches.push({name: `v${step_major}.${step_minor}-branch`}) + } +} + +// Get the current branch (we want to validate that the correct kind of release is being created) +const branch = execSync("git rev-parse --abbrev-ref HEAD").toString("utf8"); + +// Below is the configuration for semantic release +module.exports = { + branches: possible_branches, + plugins: [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + "@semantic-release/github", + [ + "@semantic-release/changelog", + { + changelogFile: "CHANGELOG.md" + } + ], + [ + "@semantic-release/git", + { + assets: [ + "CHANGELOG.md" + ], + message: "chore(release): release ${nextRelease.version}\n\n${nextRelease.notes}" + } + ], + ["@semantic-release/exec", { + "verifyReleaseCmd": "./infra/scripts/validate-release.sh ${nextRelease.type} " + branch + }] + ] +} + + diff --git a/infra/scripts/validate-release.sh b/infra/scripts/validate-release.sh new file mode 100755 index 0000000000..3ccc4f4f15 --- /dev/null +++ b/infra/scripts/validate-release.sh @@ -0,0 +1,30 @@ +# This script ensures that we don't accidentally cut the wrong kind of release on master or release branches + +if [ "$#" -ne 2 ] +then + echo "Usage: validate-release.sh [major, minor, patch] branch" + echo "Example: validate-release.sh patch master" + exit 1 +fi + +if [ "$1" = "minor" ]; then + if [ "$2" = "master" ]; then + echo "Releasing a minor version on master, looks good!" + exit 0 + else + echo "Can't release a minor version from a non-master branch! Please confirm the version you are releasing!!" + exit 1 + fi +elif [ "$1" = "patch" ]; then + if [ "$2" = "master" ]; then + echo "Can't release a patch version from master branch! Please confirm the version you are releasing!!" + exit 1 + else + echo "Releasing a patch version from a non-master branch, looks good!" + exit 0 + fi +else + echo "Not sure what kind of release is happening. Please confirm that you are creating a minor release from master + or a patch from a release branch" + exit 1 +fi \ No newline at end of file