Add Format and Lint to Makefile (feast-dev#545)

* Add Make commands for format, lint, flake8, spotless, isort, black, and refactor

* Add mypy test

* Add lint tests to CI

* Fix broken Python test

* Fix broken test for Python

* Add black to dependencies

* Remove Python Protos

* Add automatic local linting

* Update precommit names

* Add black exclusions

* Add tensorflow metadata proto generation

* Ignore tf meta directory

* Add build essentials to install make in CI

* Add exports back to __init__.py

* Add __all__ to export

* Add white space to export

* Add source to export

* Fix python export formatting

.gitignore

@@ -179,3 +179,8 @@ dmypy.json
 .flattened-pom.xml
 
 sdk/python/docs/html
+sdk/python/feast/core/
+sdk/python/feast/serving/
+sdk/python/feast/storage/
+sdk/python/feast/types/
+sdk/python/tensorflow_metadata

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: local
+    hooks:
+      - id: format
+        name: Format
+        stages: [commit]
+        language: system
+        entry: make format
+      - id: lint
+        name: Lint
+        stages: [commit]
+        language: system
+        entry: make lint

Makefile

@@ -57,7 +57,6 @@ install-python-ci-dependencies:
 compile-protos-python: install-python-ci-dependencies
 	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ feast/$(dir)/*.proto;)
 	@$(foreach dir,$(PROTO_SERVICE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/ feast/$(dir)/*.proto;)
-	cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --python_out=../sdk/python/ --mypy_out=../sdk/python/ tensorflow_metadata/proto/v0/*.proto
 
 install-python: compile-protos-python
 	pip install -e sdk/python --upgrade
@@ -157,4 +156,4 @@ build-html: clean-html
 	# Build Python SDK documentation
 	$(MAKE) compile-protos-python
 	cd 	$(ROOT_DIR)/sdk/python/docs && $(MAKE) html
-	cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python
+	cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python

infra/scripts/test-core-ingestion.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+apt-get -qq update
+apt-get -y install build-essential
+
+make lint-java
+
 infra/scripts/download-maven-cache.sh \
     --archive-uri gs://feast-templocation-kf-feast/.m2.2019-10-24.tar \
     --output-dir /root/

infra/scripts/test-end-to-end-batch.sh

@@ -22,7 +22,7 @@ This script will run end-to-end tests for Feast Core and Batch Serving.
 "
 
 apt-get -qq update
-apt-get -y install wget netcat kafkacat
+apt-get -y install wget netcat kafkacat build-essential
 
 
 echo "
@@ -233,6 +233,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f
 source ~/.bashrc
 
 # Install Feast Python SDK and test requirements
+make compile-protos-python
 pip install -qe sdk/python
 pip install -qr tests/e2e/requirements.txt
 

infra/scripts/test-end-to-end.sh

@@ -21,7 +21,7 @@ This script will run end-to-end tests for Feast Core and Online Serving.
 "
 
 apt-get -qq update
-apt-get -y install wget netcat kafkacat
+apt-get -y install wget netcat kafkacat build-essential
 
 echo "
 ============================================================
@@ -207,6 +207,7 @@ bash /tmp/miniconda.sh -b -p /root/miniconda -f
 source ~/.bashrc
 
 # Install Feast Python SDK and test requirements
+make compile-protos-python
 pip install -qe sdk/python
 pip install -qr tests/e2e/requirements.txt
 

infra/scripts/test-golang-sdk.sh

@@ -2,6 +2,8 @@
 
 set -o pipefail
 
+make lint-go
+
 cd sdk/go
 go test -v 2>&1 | tee /tmp/test_output
 TEST_EXIT_CODE=$?

infra/scripts/test-python-sdk.sh

@@ -5,7 +5,10 @@ set -e
 # Default artifact location setting in Prow jobs
 LOGS_ARTIFACT_PATH=/logs/artifacts
 
-cd sdk/python
-pip install -r requirements-ci.txt
+pip install -r sdk/python/requirements-ci.txt
+make compile-protos-python
+make lint-python
+
+cd sdk/python/
 pip install -e .
 pytest --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml

sdk/go/client.go

@@ -53,7 +53,7 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures
 	return &OnlineFeaturesResponse{RawResponse: resp}, err
 }
 
-// GetInfo gets information about the feast serving instance this client is connected to.
+// GetFeastServingInfo gets information about the feast serving instance this client is connected to.
 func (fc *GrpcClient) GetFeastServingInfo(ctx context.Context, in *serving.GetFeastServingInfoRequest) (
 	*serving.GetFeastServingInfoResponse, error) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, "get_info")
@@ -62,7 +62,7 @@ func (fc *GrpcClient) GetFeastServingInfo(ctx context.Context, in *serving.GetFe
 	return fc.cli.GetFeastServingInfo(ctx, in)
 }
 
-// Closes the grpc connection.
+// Close the grpc connection.
 func (fc *GrpcClient) Close() error {
 	return fc.conn.Close()
-}
+}