Merge remote-tracking branch 'upstream/master' into dynamodb-improvem…

…ents * upstream/master: (103 commits) refactor: Move create cluster script ci: Add validation for upper case PR titles Switch order of views to keep/delete to fix apply logging ci: Fix broken Prow plugin which ignores config.yaml ci: Add PR linting to ensure conventional commits are followed for PR titles Add owner field to FeatureService (feast-dev#2321) Disable Redis cluster tests (feast-dev#2327) Use PR title for commit message Add redis cluster initialziation to master github workflow (feast-dev#2317) Fix materialize bug with RedisCluster (feast-dev#2311) Add support for Dask during historical retrieval (feast-dev#1954) downgrade datatypes to 8 Update pom.xml to 0.18.2 SNAPSHOT Update changelog for 0.18.1 and update pom.xml Update helm chart version Don't require `snowflake` to always be installed (feast-dev#2309) Fixing the Java helm charts and adding a demo tutorial on how to use them (feast-dev#2298) Include infra objects in registry dump and fix Infra's from_proto (feast-dev#2295) Remove old flag warning with the python feature server (feast-dev#2300) podAnnotations Values in the feature-server chart (feast-dev#2304) ...
aht · Feb 23, 2022 · bb1875f · bb1875f
2 parents b4a011a + 9221624
commit bb1875f
Show file tree

Hide file tree

Showing 339 changed files with 10,913 additions and 9,117 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: 'kind/bug, priority/p2'
 assignees: ''
 
 ---

diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: 'kind/feature'
 assignees: ''
 
 ---

diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml
@@ -0,0 +1,24 @@
+name: lint-pr
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  validate-title:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v4
+        with:
+          # Must use uppercase
+          subjectPattern: ^(?=[A-Z]).+$
+          subjectPatternError: |
+            The subject "{subject}" found in the pull request title "{title}"
+            didn't match the configured pattern. Please ensure that the subject
+            starts with an uppercase character.
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -120,11 +120,20 @@ jobs:
         run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
+      - name: Setup Redis Cluster
+        run: |
+          docker pull vishnunair/docker-redis-cluster:latest
+          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
       - name: Test python
         env:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
+          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
+          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
+          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
+          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
         run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
@@ -140,6 +149,11 @@ jobs:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
+          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
+          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
+          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
+          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
         run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
       - name: Upload Benchmark Artifact to S3
         run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmarks

diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -145,12 +145,21 @@ jobs:
         run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
+      - name: Setup Redis Cluster
+        run: |
+          docker pull vishnunair/docker-redis-cluster:latest
+          docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
       - name: Test python
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
         env:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
+          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
+          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
+          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
+          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
         run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -103,7 +103,7 @@ jobs:
     runs-on:  ubuntu-latest
     needs: get-version
     env:
-      HELM_VERSION: v2.17.0
+      HELM_VERSION: v3.8.0
       VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
     steps:
       - uses: actions/checkout@v2
@@ -144,5 +144,63 @@ jobs:
           python3 setup.py sdist bdist_wheel
           python3 -m twine upload --verbose dist/*
 
-    # TODO(adchia): publish java sdk once maven repo is updated
-    # See https://github.com/feast-dev/feast-java/blob/master/.github/workflows/release.yml#L104
+  publish-python-sdk-no-telemetry:
+    runs-on: ubuntu-latest
+    needs: get-version
+    env:
+      TWINE_USERNAME: __token__
+      TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+    container: python:3.7
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install pip-tools
+        run: pip install pip-tools
+      - name: Install dependencies
+        run: make install-python-ci-dependencies PYTHON=3.7
+      - name: Publish Python Package
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.get-version.outputs.version_without_prefix }}
+        run: |
+          cd sdk/python
+          sed -i 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py
+          sed -i 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py
+          python3 -m pip install --user --upgrade setuptools wheel twine
+          python3 setup.py sdist bdist_wheel
+          python3 -m twine upload --verbose dist/*
+
+  publish-java-sdk:
+    container: maven:3.6-jdk-11
+    runs-on: ubuntu-latest
+    needs: get-version
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: '11'
+          java-package: jdk
+          architecture: x64
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+          architecture: 'x64'
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-it-maven-
+      - name: Publish java sdk
+        env:
+          VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
+          GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }}
+          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+          MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }}
+        run: |
+          echo -n "$GPG_PUBLIC_KEY" > /root/public-key
+          echo -n "$GPG_PRIVATE_KEY" > /root/private-key
+          mkdir -p /root/.m2/
+          echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml
+          infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -42,6 +42,12 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
+        env:
+          SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
+          SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
+          SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
+          SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
+          SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
         run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1

diff --git a/.prow.yaml b/.prow.yaml
@@ -1,102 +1,4 @@
-presubmits:
-- name: test-core-and-ingestion
-  decorate: true
-  spec:
-    containers:
-    - image: maven:3.6-jdk-11
-      command: ["infra/scripts/test-java-core-ingestion.sh"]
-      resources:
-        requests:
-          cpu: "2000m"
-          memory: "1536Mi"
-  skip_branches:
-  - ^v0\.(3|4)-branch$
-
-- name: test-core-and-ingestion-java-8
-  decorate: true
-  always_run: true
-  spec:
-    containers:
-      - image: maven:3.6-jdk-8
-        command: ["infra/scripts/test-java-core-ingestion.sh"]
-        resources:
-          requests:
-            cpu: "2000m"
-            memory: "1536Mi"
-  branches:
-  - ^v0\.(3|4)-branch$
-
-- name: test-serving
-  decorate: true
-  spec:
-    containers:
-    - image: maven:3.6-jdk-11
-      command: ["infra/scripts/test-java-serving.sh"]
-  skip_branches:
-  - ^v0\.(3|4)-branch$
-
-- name: test-serving-java-8
-  decorate: true
-  always_run: true
-  spec:
-    containers:
-      - image: maven:3.6-jdk-8
-        command: ["infra/scripts/test-java-serving.sh"]
-  branches:
-  - ^v0\.(3|4)-branch$
-
-- name: test-java-sdk
-  decorate: true
-  spec:
-    containers:
-    - image: maven:3.6-jdk-11
-      command: ["infra/scripts/test-java-sdk.sh"]
-  skip_branches:
-  - ^v0\.(3|4)-branch$
-
-- name: test-java-sdk-java-8
-  decorate: true
-  always_run: true
-  spec:
-    containers:
-      - image: maven:3.6-jdk-8
-        command: ["infra/scripts/test-java-sdk.sh"]
-  branches:
-  - ^v0\.(3|4)-branch$
-
-- name: test-golang-sdk
-  decorate: true
-  spec:
-    containers:
-    - image: golang:1.13
-      command: ["infra/scripts/test-golang-sdk.sh"]
-
 postsubmits:
-- name: publish-python-sdk
-  decorate: true
-  spec:
-    containers:
-    - image: python:3
-      command:
-      - sh
-      - -c
-      - |
-        make package-protos && make compile-protos-python && infra/scripts/publish-python-sdk.sh \
-          --directory-path sdk/python --repository pypi
-      volumeMounts:
-      - name: pypirc
-        mountPath: /root/.pypirc
-        subPath: .pypirc
-        readOnly: true
-    volumes:
-    - name: pypirc
-      secret:
-        secretName: pypirc
-  branches:
-  # Filter on tags with semantic versioning, prefixed with "v"
-  # https://github.com/semver/semver/issues/232
-  - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
-
 - name: publish-java-sdk
   decorate: true
   spec:
@@ -128,31 +30,3 @@ postsubmits:
   branches:
   # Filter on tags with semantic versioning, prefixed with "v".
   - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
-
-- name: publish-java-8-sdk
-  decorate: true
-  spec:
-    containers:
-      - image: maven:3.6-jdk-8
-        command:
-          - bash
-          - -c
-          - infra/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1}
-        volumeMounts:
-          - name: gpg-keys
-            mountPath: /etc/gpg
-            readOnly: true
-          - name: maven-settings
-            mountPath: /root/.m2/settings.xml
-            subPath: settings.xml
-            readOnly: true
-    volumes:
-      - name: gpg-keys
-        secret:
-          secretName: gpg-keys
-      - name: maven-settings
-        secret:
-          secretName: maven-settings
-  branches:
-  # Filter on tags with semantic versioning, prefixed with "v". v0.3 and v0.4 only.
-  - ^v0\.(3|4)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$
diff --git a/.prow/config.yaml b/.prow/config.yaml
@@ -64,6 +64,9 @@ tide:
   merge_method:
     feast-dev/feast: squash
     feast-dev/feast-spark: squash
+  merge_commit_template:
+    feast-dev/feast:
+      title: '{{.Title}}'
   blocker_label: merge-blocker
   squash_label: tide/squash
 
diff --git a/.prow/plugins.yaml b/.prow/plugins.yaml
@@ -34,6 +34,8 @@ config_updater:
   maps:
     .prow/config.yaml:
       name: config
+    .prow.yaml:
+      name: job-config
 
 external_plugins:
   feast-dev/feast: