Use pipenv

Signed-off-by: Yusuke Nishioka <yusuke.nishioka.0713@gmail.com>
feast-dev · Nov 26, 2021 · da9c425 · da9c425
1 parent 54d0f3a
commit da9c425
Show file tree

Hide file tree

Showing 15 changed files with 10,392 additions and 26 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -107,14 +107,17 @@ jobs:
           key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pipenv
+        run: pip install pipenv
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python
         env:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
-        run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
+        run: pipenv run pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes tests --integration --durations=5
+        working-directory: sdk/python
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:
@@ -129,6 +132,7 @@ jobs:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
-        run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
+        run: pipenv run pytest --verbose --color=yes tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
+        working-directory: sdk/python
       - name: Upload Benchmark Artifact to S3
         run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmarks
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -5,6 +5,8 @@ on: [push, pull_request]
 jobs:
   lint-python:
     runs-on: [ubuntu-latest]
+    env:
+      PYTHON: 3.7
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
@@ -27,9 +29,12 @@ jobs:
             ${{ steps.pip-cache.outputs.dir }}
             /opt/hostedtoolcache/Python
             /Users/runner/hostedtoolcache/Python
+            ~/.local/share/virtualenvs
           key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pipenv
+        run: pip install pipenv
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Lint python

diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -131,14 +131,17 @@ jobs:
           key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pipenv
+        run: pip install pipenv
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python
         env:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
-        run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
+        run: pipenv run pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes tests --integration --durations=5
+        working-directory: sdk/python
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:
@@ -153,6 +156,7 @@ jobs:
           FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
           FEAST_USAGE: "False"
           IS_TEST: "True"
-        run: pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
+        run: pipenv run pytest --verbose --color=yes tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5
+        working-directory: sdk/python
       - name: Upload Benchmark Artifact to S3
         run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmarks
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -130,6 +130,8 @@ jobs:
     container: python:3.7
     steps:
       - uses: actions/checkout@v2
+      - name: Install pipenv
+        run: pip install pipenv
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Publish Python Package

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -34,15 +34,17 @@ jobs:
             ${{ steps.pip-cache.outputs.dir }}
             /opt/hostedtoolcache/Python
             /Users/runner/hostedtoolcache/Python
+            ~/.local/share/virtualenvs
           key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pipenv
+        run: pip install pipenv
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
-        run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests
-      - name: Ensure conflict-free dependencies
-        run: FEAST_USAGE=False pip-compile --dry-run sdk/python/setup.py --extra ci
+        run: FEAST_USAGE=False pipenv run pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes tests
+        working-directory: sdk/python
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

diff --git a/Makefile b/Makefile
@@ -39,8 +39,18 @@ install-ci-dependencies: install-python-ci-dependencies install-java-ci-dependen
 
 # Python SDK
 
-install-python-ci-dependencies:
-	pip install -e "sdk/python[ci]"
+pipfile:
+	if [ ! -e sdk/python/Pipfile ]; then\
+		cd ${ROOT_DIR}/sdk/python;\
+		ln -s pipfiles/$(PYTHON)/Pipfile;\
+		ln -s pipfiles/$(PYTHON)/Pipfile.lock;\
+	fi
+
+install-python-ci-dependencies: pipfile
+	cd ${ROOT_DIR}/sdk/python; pipenv sync --dev
+
+update-lock: pipfile
+	cd ${ROOT_DIR}/sdk/python; pipenv update --dev
 
 package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
@@ -49,39 +59,39 @@ compile-protos-python:
 	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),cd ${ROOT_DIR}/protos; python -m grpc_tools.protoc -I. --grpc_python_out=../sdk/python/feast/protos/ --python_out=../sdk/python/feast/protos/ --mypy_out=../sdk/python/feast/protos/ feast/$(dir)/*.proto;)
 	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -I@ sed -i.bak 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
 
-install-python:
-	python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver
+install-python: pipfile
+	cd ${ROOT_DIR}/sdk/python; pipenv sync
 
 benchmark-python:
-	FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
+	cd ${ROOT_DIR}/sdk/python; FEAST_USAGE=False IS_TEST=True pipenv run python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data tests
 
 benchmark-python-local:
-	FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests
+	cd ${ROOT_DIR}/sdk/python; FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True pipenv run python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data tests
 
 test-python:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 sdk/python/tests
+	cd ${ROOT_DIR}/sdk/python; FEAST_USAGE=False IS_TEST=True pipenv run python -m pytest -n 8 tests
 
 test-python-integration:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
+	cd ${ROOT_DIR}/sdk/python; FEAST_USAGE=False IS_TEST=True pipenv run python -m pytest -n 8 --integration tests
 
 test-python-universal-local:
-	FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
+	cd ${ROOT_DIR}/sdk/python; FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True pipenv run python -m pytest -n 8 --integration --universal tests
 
 test-python-universal:
-	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
+	cd ${ROOT_DIR}/sdk/python; FEAST_USAGE=False IS_TEST=True pipenv run python -m pytest -n 8 --integration --universal tests
 
 format-python:
 	# Sort
-	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/
+	cd ${ROOT_DIR}/sdk/python; pipenv run python -m isort feast/ tests/
 
 	# Format
-	cd ${ROOT_DIR}/sdk/python; python -m black --target-version py37 feast tests
+	cd ${ROOT_DIR}/sdk/python; pipenv run python -m black --target-version py37 feast tests
 
 lint-python:
-	cd ${ROOT_DIR}/sdk/python; python -m mypy feast/ tests/
-	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ --check-only
-	cd ${ROOT_DIR}/sdk/python; python -m flake8 feast/ tests/
-	cd ${ROOT_DIR}/sdk/python; python -m black --check feast tests
+	cd ${ROOT_DIR}/sdk/python; pipenv run python -m mypy feast/ tests/
+	cd ${ROOT_DIR}/sdk/python; pipenv run python -m isort feast/ tests/ --check-only
+	cd ${ROOT_DIR}/sdk/python; pipenv run python -m flake8 feast/ tests/
+	cd ${ROOT_DIR}/sdk/python; pipenv run python -m black --check feast tests
 
 # Java
 

diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
@@ -7,7 +7,7 @@ infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir
 apt-get update && apt-get install -y redis-server postgresql libpq-dev
 
 make build-java-no-tests REVISION=develop
-python -m pip install --upgrade pip setuptools wheel
+python -m pip install --upgrade pip setuptools wheel pipenv
 make install-python
 python -m pip install -qr tests/requirements.txt
 export FEAST_USAGE="False"

diff --git a/infra/scripts/test-integration.sh b/infra/scripts/test-integration.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-python -m pip install --upgrade pip setuptools wheel
+python -m pip install --upgrade pip setuptools wheel pipenv
 make install-python
 python -m pip install -qr tests/requirements.txt
 

diff --git a/sdk/python/pipfiles/3.7/Pipfile b/sdk/python/pipfiles/3.7/Pipfile
@@ -0,0 +1,13 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+feast = {editable = true, path = "../.."}
+
+[dev-packages]
+feast = {editable = true, extras = ["dev"], path = "../.."}
+
+[requires]
+python_version = "3.7"