Merge branch 'master' into fixPRTests

.github/workflows/linter.yml

@@ -15,6 +15,11 @@ jobs:
         with:
           python-version: "3.7"
           architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.7
       - name: Upgrade pip version
         run: |
           pip install --upgrade "pip>=21.3.1"
@@ -35,16 +40,30 @@ jobs:
       - name: Install pip-tools
         run: pip install pip-tools
       - name: Install dependencies
-        run: make install-python-ci-dependencies
+        run: |
+          make compile-protos-go
+          make install-python-ci-dependencies
       - name: Lint python
         run: make lint-python
 
   lint-go:
-    container: gcr.io/kf-feast/feast-ci:latest
-    runs-on: [ubuntu-latest]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.7
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade "pip>=21.3.1"
       - name: Install dependencies
-        run: make install-go-ci-dependencies
+        run:  make install-go-ci-dependencies
       - name: Lint go
         run: make lint-go

.github/workflows/master_only.yml

@@ -54,13 +54,14 @@ jobs:
           docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
     outputs:
       DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
-  integration-test-python:
+  integration-test-python-and-go:
     needs: build-lambda-docker-image
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         python-version: [ 3.7, 3.8, 3.9 ]
+        go-version: [ 1.17.0 ]
         os: [ ubuntu-latest ]
     env:
       OS: ${{ matrix.os }}
@@ -73,6 +74,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v0
         with:
@@ -113,13 +119,12 @@ jobs:
       - name: Start Redis
         uses: supercharge/redis-github-action@1.4.0
         with:
-          redis-version: ${{ matrix.redis-version }}
           redis-port: 12345
       - name: Setup Redis Cluster
         run: |
           docker pull vishnunair/docker-redis-cluster:latest
           docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
-      - name: Test python
+      - name: Test python and go
         env:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-lambda-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"

.github/workflows/pr_integration_tests.yml

@@ -144,7 +144,9 @@ jobs:
       - name: Install pip-tools
         run: pip install pip-tools
       - name: Install dependencies
-        run: make install-python-ci-dependencies
+        run: |
+          make compile-protos-go
+          make install-python-ci-dependencies
       - name: Setup Redis Cluster
         run: |
           docker pull vishnunair/docker-redis-cluster:latest

.github/workflows/release.yml

@@ -40,4 +40,4 @@ jobs:
     - name: Release
       if: github.event.inputs.dry_run == 'false'
       run: |
-        npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release
+        npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release

.github/workflows/unit_tests.yml

@@ -20,6 +20,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.7
       - name: Upgrade pip version
         run: |
           pip install --upgrade "pip>=21.3.1"
@@ -40,7 +45,9 @@ jobs:
       - name: Install pip-tools
         run: pip install pip-tools
       - name: Install dependencies
-        run: make install-python-ci-dependencies
+        run: |
+          make compile-protos-go
+          make install-python-ci-dependencies
       - name: Test Python
         env:
           SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
@@ -61,9 +68,21 @@ jobs:
 
   unit-test-go:
     runs-on: ubuntu-latest
-    container: gcr.io/kf-feast/feast-ci:latest
     steps:
       - uses: actions/checkout@v2
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Upgrade pip version
+        run: |
+          pip install --upgrade "pip>=21.3.1"
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.7
       - name: Install dependencies
         run: make install-go-ci-dependencies
       - name: Compile protos

.gitignore

@@ -186,6 +186,7 @@ dmypy.json
 sdk/python/docs/html
 sdk/python/feast/protos/
 sdk/go/protos/
+go/protos/
 
 #benchmarks
 .benchmarks
@@ -218,3 +219,5 @@ ui/.vercel
 **/yarn-debug.log*
 **/yarn-error.log*
 
+# Go subprocess binaries (built during feast pip package building)
+sdk/python/feast/binaries/

.releaserc.js

@@ -74,4 +74,3 @@ module.exports = {
         }],
     ]
 }
-

Makefile

@@ -16,8 +16,6 @@
 
 ROOT_DIR 	:= $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 MVN := mvn -f java/pom.xml ${MAVEN_EXTRA_OPTS}
-PROTO_TYPE_SUBDIRS = core serving types storage
-PROTO_SERVICE_SUBDIRS = core serving
 OS := linux
 ifeq ($(shell uname -s), Darwin)
 	OS = osx
@@ -35,13 +33,11 @@ protos: compile-protos-go compile-protos-python compile-protos-docs
 
 build: protos build-java build-docker build-html
 
-install-ci-dependencies: install-python-ci-dependencies install-java-ci-dependencies install-go-ci-dependencies
-
 # Python SDK
 
-install-python-ci-dependencies:
+install-python-ci-dependencies: install-go-ci-dependencies
 	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
-	cd sdk/python && python setup.py develop
+	cd sdk/python && COMPILE_GO=true python setup.py develop
 
 lock-python-ci-dependencies:
 	cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt
@@ -50,8 +46,7 @@ package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
 
 compile-protos-python:
-	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
-	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -I@ sed -i.bak 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
+	python setup.py build_python_protos
 
 install-python:
 	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt
@@ -72,12 +67,23 @@ test-python:
 test-python-integration:
 	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
 
+test-python-universal-contrib:
+	PYTHONPATH='.' FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.contrib_repo_configuration FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
+
 test-python-universal-local:
 	FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
 
 test-python-universal:
 	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
 
+test-python-go-server:
+	go build -o ${ROOT_DIR}/sdk/python/feast/binaries/goserver github.com/feast-dev/feast/go/cmd/goserver
+	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserver sdk/python/tests
+
+test-python-go-server-lifecycle:
+	go build -o ${ROOT_DIR}/sdk/python/feast/binaries/goserver github.com/feast-dev/feast/go/cmd/goserver
+	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --goserverlifecycle sdk/python/tests
+
 format-python:
 	# Sort
 	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/
@@ -120,20 +126,25 @@ build-java-no-tests:
 # Go SDK
 
 install-go-ci-dependencies:
-	go get -u github.com/golang/protobuf/protoc-gen-go
-	go get -u golang.org/x/lint/golint
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0
+	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0
+
+compile-protos-go: install-go-ci-dependencies
+	pip install grpcio-tools==1.34.0
+	python sdk/python/setup.py build_go_protos
 
-compile-protos-go:
-	$(foreach dir,types serving core storage,cd ${ROOT_DIR}/protos; protoc -I/usr/local/include -I. --go_out=plugins=grpc,paths=source_relative:../sdk/go/protos feast/$(dir)/*.proto;)
+compile-go-feature-server: compile-protos-go
+	go mod tidy
+	go build -o ${ROOT_DIR}/sdk/python/feast/binaries/goserver github.com/feast-dev/feast/go/cmd/goserver
 
-test-go:
-	cd ${ROOT_DIR}/sdk/go; go test ./...
+test-go: install-go-ci-dependencies
+	go test ./...
 
 format-go:
-	cd ${ROOT_DIR}/sdk/go; gofmt -s -w *.go
+	gofmt -s -w go/
 
-lint-go:
-	cd ${ROOT_DIR}/sdk/go; go vet
+lint-go: compile-protos-go
+	go vet ./go/internal/feast ./go/cmd/goserver
 
 # Docker
 
@@ -198,4 +209,4 @@ build-sphinx: compile-protos-python
 	cd 	$(ROOT_DIR)/sdk/python/docs && $(MAKE) build-api-source
 
 build-templates:
-	python infra/scripts/compile-templates.py
+	python infra/scripts/compile-templates.py

docs/how-to-guides/adding-or-reusing-tests.md

@@ -161,6 +161,13 @@ The key fixtures are the `environment` and `universal_data_sources` fixtures, wh
 * In `repo_configuration.py` add a new`IntegrationTestRepoConfig` or two (depending on how many online stores you want to test).
 * Run the full test suite with `make test-python-integration.`
 
+### Including a new offline / online store in the main Feast repo from external plugins with community maintainers.
+
+* This folder is for plugins that are officially maintained with community owners. Place the APIs in feast/infra/offline_stores/contrib/.
+* Extend `data_source_creator.py` for your offline store and implement the required APIs.
+* In `contrib_repo_configuration.py` add a new `IntegrationTestRepoConfig` (depending on how many online stores you want to test).
+* Run the test suite on the contrib test suite with `make test-python-contrib-universal`.
+
 ### To include a new online store
 
 * In `repo_configuration.py` add a new config that maps to a serialized version of configuration you need in `feature_store.yaml` to setup the online store.

go.mod

@@ -1,39 +1,29 @@
 module github.com/feast-dev/feast
 
+go 1.17
+
 require (
-	github.com/Masterminds/goutils v1.1.0 // indirect
-	github.com/Masterminds/semver v1.5.0 // indirect
-	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
-	github.com/feast-dev/feast/sdk/go v0.0.0-20200516052424-09ff3dda724c // indirect
 	github.com/ghodss/yaml v1.0.0
-	github.com/gogo/protobuf v1.3.1 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/golang/mock v1.2.0
+	github.com/go-redis/redis/v8 v8.11.4
 	github.com/golang/protobuf v1.5.2
-	github.com/google/go-cmp v0.5.5
-	github.com/huandu/xstrings v1.2.0 // indirect
-	github.com/lyft/protoc-gen-validate v0.1.0 // indirect
-	github.com/mitchellh/copystructure v1.0.0 // indirect
-	github.com/mitchellh/go-homedir v1.1.0
-	github.com/mwitkow/go-proto-validators v0.2.0 // indirect
-	github.com/pseudomuto/protoc-gen-doc v1.3.0 // indirect
-	github.com/pseudomuto/protokit v0.2.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/spf13/cobra v0.0.4
-	github.com/spf13/viper v1.4.0
-	github.com/woop/protoc-gen-doc v1.3.0 // indirect
-	go.opencensus.io v0.22.3 // indirect
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
-	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
-	golang.org/x/tools v0.1.8 // indirect
-	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
-	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect
-	gopkg.in/yaml.v2 v2.2.4
-	istio.io/gogo-genproto v0.0.0-20191212213402-78a529a42cd8 // indirect
-	istio.io/tools v0.0.0-20191228030621-c4eb6a11039c // indirect
+	github.com/google/uuid v1.1.2
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/spaolacci/murmur3 v1.1.0
+	github.com/stretchr/testify v1.7.0
+	google.golang.org/grpc v1.44.0
+	google.golang.org/protobuf v1.27.1
 )
 
-go 1.13
+require (
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	golang.org/x/text v0.3.6 // indirect
+	google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)