fix: python dependency specification (#2483)

Co-authored-by: universalmind303 <cory.grinstead@gmail.com>

.github/workflows/ci.yaml

@@ -138,6 +138,7 @@ jobs:
             !target/**/glaredb
             !target/**/pgprototest
           key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
+      - run: just python venv
       - run: just python build
       - run: just python test
 

bindings/python/justfile

@@ -1,29 +1,25 @@
 set fallback
-VENV := env_var_or_default("VENV", "../../.venv")
+VENV := env_var_or_default("VENV", ".venv")
 VENV_BIN := VENV / "bin"
 
 @venv:
-  python3 -c "import virtualenv" || python3 -m pip --quiet install virtualenv
-  python3 -m virtualenv {{VENV}} --quiet
-
-## Set up virtual environment and install requirements
-@requirements: venv
-  {{VENV_BIN}}/python -m pip --quiet install --upgrade pip
-  {{VENV_BIN}}/pip --quiet install -r requirements.txt
+  if python3 -c "import virtualenv"; then python3 -m virtualenv {{VENV}}; else python3 -m venv {{VENV}}; fi
+  {{VENV_BIN}}/python -m pip install --upgrade pip
+  {{VENV_BIN}}/pip install -r requirements.txt
 
 ## Compile and install py-glaredb for development
-build *args: requirements
+build *args:
   @unset CONDA_PREFIX
   {{VENV_BIN}}/maturin develop {{args}}
 
-test: requirements
-  {{VENV_BIN}}/pytest -s
+test:
+  {{VENV_BIN}}/pytest --rootdir={{justfile_directory()}}
 
 ## Run autoformatting and linting
-fmt: venv
+fmt:
   {{VENV_BIN}}/ruff .
   {{VENV_BIN}}/mypy
   cargo fmt --all
 
-example path: requirements
+example path:
   {{VENV_BIN}}/python examples/{{path}}.py

bindings/python/pyproject.toml

@@ -6,13 +6,32 @@ build-backend = "maturin"
 name = "glaredb"
 description = "GlareDB is a fast SQL database for querying and analyzing distributed data."
 readme = "README.md"
+version = "0.0.0"
 requires-python = ">=3.7"
 classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
 
+
+[project.optional-dependencies]
+great-expectations = [
+  "great_expectations",
+  "great_expectations[postgresql]",
+]
+pandas = [
+  "pandas",
+  "pandasai",
+]
+polars = [
+  "polars",
+]
+pyarrow = [
+  "pyarrow",
+]
+
+
 [project.urls]
 Homepage = "https://glaredb.com"
 Documentation = "https://docs.glaredb.com"
@@ -27,4 +46,4 @@ line-length = 88
 fix = true
 ignore = [
     "F841" # unused variables
-]
+]

bindings/python/requirements.txt

@@ -1,14 +1,11 @@
-maturin==1.0.1
-virtualenv==20.23.1
-ruff==0.0.275
+# this requirements.txt is for development deps only
+maturin==1.4.0
 black==23.3.0
 blackdoc==0.3.8
 mypy==1.3.0
+pytest
+ruff==0.0.275
 typos==1.15
-pyarrow
 pandas
 polars
-pytest
-pandasai
-great_expectations[postgresql]
-great_expectations
+pyarrow

justfile

@@ -118,16 +118,15 @@ protoc:
     rm protoc.zip
   fi
 
-
 # Installs python dependencies for testing
 venv:
-  python3 -c "import virtualenv" || python3 -m pip --quiet install virtualenv
-  python3 -m virtualenv .venv --quiet
+  if python3 -c "import virtualenv"; then python3 -m virtualenv {{VENV}}; else python3 -m venv {{VENV}}; fi
+  {{VENV_BIN}}/python -m pip install --upgrade pip
   {{VENV_BIN}}/python -m pip install poetry
-  {{VENV_BIN}}/poetry -C tests install
 
 # Runs pytest in the tests directory.
 pytest *args:
+  {{VENV_BIN}}/poetry -C tests install --no-root
   {{VENV_BIN}}/poetry -C tests lock --no-update
   {{VENV_BIN}}/poetry -C tests run pytest --rootdir={{invocation_directory()}}/tests {{ if args == "" {'tests'} else {args} }}