Use pip-tools to lock dependencies

Signed-off-by: Yusuke Nishioka <yusuke.nishioka.0713@gmail.com>
feast-dev · Dec 6, 2021 · 56751ee · 56751ee
1 parent d5ba5cf
commit 56751ee
Show file tree

Hide file tree

Showing 14 changed files with 2,268 additions and 11 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -104,9 +104,11 @@ jobs:
             ${{ steps.pip-cache.outputs.dir }}
             /opt/hostedtoolcache/Python
             /Users/runner/hostedtoolcache/Python
-          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -5,6 +5,8 @@ on: [push, pull_request]
 jobs:
   lint-python:
     runs-on: [ubuntu-latest]
+    env:
+      PYTHON: 3.7
     steps:
       - uses: actions/checkout@v2
       - name: Setup Python
@@ -27,9 +29,11 @@ jobs:
             ${{ steps.pip-cache.outputs.dir }}
             /opt/hostedtoolcache/Python
             /Users/runner/hostedtoolcache/Python
-          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Lint python

diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -128,9 +128,11 @@ jobs:
             ${{ steps.pip-cache.outputs.dir }}
             /opt/hostedtoolcache/Python
             /Users/runner/hostedtoolcache/Python
-          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -130,6 +130,8 @@ jobs:
     container: python:3.7
     steps:
       - uses: actions/checkout@v2
+      - name: Install pip-tools
+        run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Publish Python Package

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -34,15 +34,15 @@ jobs:
             ${{ steps.pip-cache.outputs.dir }}
             /opt/hostedtoolcache/Python
             /Users/runner/hostedtoolcache/Python
-          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }}
+          key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
           restore-keys: |
             ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
+      - name: Install pip-tools
+        run: pip install pip-tools
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test Python
         run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests
-      - name: Ensure conflict-free dependencies
-        run: FEAST_USAGE=False pip-compile --dry-run sdk/python/setup.py --extra ci
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:
@@ -63,4 +63,4 @@ jobs:
       - name: Compile protos
         run: make compile-protos-go
       - name: Test
-        run: make test-go
+        run: make test-go
diff --git a/Makefile b/Makefile
@@ -40,7 +40,11 @@ install-ci-dependencies: install-python-ci-dependencies install-java-ci-dependen
 # Python SDK
 
 install-python-ci-dependencies:
-	pip install -e "sdk/python[ci]"
+	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-ci-requirements.txt
+	cd sdk/python && python setup.py develop
+
+lock-python-ci-dependencies:
+	cd sdk/python && python -m piptools compile -U --extra ci --output-file requirements/py$(PYTHON)-ci-requirements.txt
 
 package-protos:
 	cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos
@@ -50,7 +54,11 @@ compile-protos-python:
 	@$(foreach dir,$(PROTO_TYPE_SUBDIRS),grep -rli 'from feast.$(dir)' sdk/python/feast/protos | xargs -I@ sed -i.bak 's/from feast.$(dir)/from feast.protos.feast.$(dir)/g' @;)
 
 install-python:
-	python -m pip install -e sdk/python -U --use-deprecated=legacy-resolver
+	cd sdk/python && python -m piptools sync requirements/py$(PYTHON)-requirements.txt
+	cd sdk/python && python setup.py develop
+
+lock-python-dependencies:
+	cd sdk/python && python -m piptools compile -U --output-file requirements/py$(PYTHON)-requirements.txt
 
 benchmark-python:
 	FEAST_USAGE=False IS_TEST=True python -m pytest --integration --benchmark  --benchmark-autosave --benchmark-save-data sdk/python/tests

diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh
@@ -7,7 +7,7 @@ infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir
 apt-get update && apt-get install -y redis-server postgresql libpq-dev
 
 make build-java-no-tests REVISION=develop
-python -m pip install --upgrade pip setuptools wheel
+python -m pip install --upgrade pip setuptools wheel pip-tools
 make install-python
 python -m pip install -qr tests/requirements.txt
 export FEAST_USAGE="False"

diff --git a/infra/scripts/test-integration.sh b/infra/scripts/test-integration.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-python -m pip install --upgrade pip setuptools wheel
+python -m pip install --upgrade pip setuptools wheel pip-tools
 make install-python
 python -m pip install -qr tests/requirements.txt