From 5231bac2f7612882a1e6d9047ad4a6bc1cbf9330 Mon Sep 17 00:00:00 2001 From: Lokesh Rangineni <lokeshforjava@gmail.com> Date: Wed, 12 Jun 2024 15:46:39 -0400 Subject: [PATCH] Draft changes to add remote online store to feast. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Adding the integration test and remote online creator class so that it will fit into existing integration testing framework. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Fix after rebase Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Removing the RemoteOnlineStoreCreator and adding custom integration test case. Incorporating the code review comments. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> reformatting the code, removing unnecessary braces. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Trying to fix the errors reported in make lint-python Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Ran the command make format-python and trying to see if it fixes the lint errors. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> increasing the server start timeout to see if it fixes the integration test cases. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> checking changes after make format-python Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> trying to see if this fixes the PR integrationt test failure. Signed-off-by: Lokesh Rangineni <lokeshemail@email.com> Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> checking in the changes for make format-python Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Upgrading python version to 3.11, adding support for 3.11 as well. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> chore: Bump macOS runners to macos-13 (#4152) bump macos runner to 13 Signed-off-by: tokoko <togurg14@freeuni.edu.ge> Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> chore: Use pixi to lock python dependencies in a single command (#4114) use pixi to lock python dependencies in a single command Signed-off-by: tokoko <togurg14@freeuni.edu.ge> Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> feat: List all feature views (#4256) * feature: Adding type to base feature view Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fixed linter Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fixed type and meta Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding new listing Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * cleaning up changes Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * reverting FV proto Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * doing simple way Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * added a test Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated to add warnings Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> --------- Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> feat: Adding vector search for sqlite (#4176) * feat: Adding vector search for sqlite Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding the sqlite_vss dependency Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * linter Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * latest progress Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * uploading latest progress Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated function Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding configuration Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding current progress Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updating requirements files Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * moving to sqlite-vec Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updating sqlite.py Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * checking in progress Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated test type Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * got the initialization working, nice Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * checking in progress from last night Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * removing unnecessary stuff Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fixing merge conflicts Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * removing files changed accidentally] Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * uploading current progress...things run but need to update the virtual table insertion Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * linted Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding working notes Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * found a bug, original feature_store.py was only grabbing first feature view, adjusted Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * cant use a string have to verify it is a proper FeatureView object Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated got it working, need to fix some other stuff still Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * working Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * linter Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fixing some type issues Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fixed typing and lint issues Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated dependencies Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fix for pixi and updating requirements Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fixed type Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * linter Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * testing sqlite_vec import Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding minimal example test Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * lint Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * testing raw sqlite Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * Printing package version * printing version Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated requirements * rebuilding requirments Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * only going to run this on 3.10 for now Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated docs for sqlite caveats Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding reason Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * skipping Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated tests Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * removing print Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * added method call Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * added prubt Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * added print Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * removing print Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * adding check in sqlite Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * missed an = Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * still running on 3.11 Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * typo Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fix Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * fix Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * updated setup and docs Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> * renamed things Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> --------- Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> squashing the last 15 commits to one. Merge branch 'master' into feature/adding-remote-onlinestore-rebase Adding documentation and incorporating code review comment. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Adding documentation and incorporating code review comment. Signed-off-by: Lokesh Rangineni <lokeshforjava@gmail.com> Merge remote-tracking branch 'fork/feature/adding-remote-onlinestore-rebase' into feature/adding-remote-onlinestore-rebase --- .github/workflows/pr_integration_tests.yml | 2 +- Makefile | 6 - docs/reference/alpha-vector-database.md | 18 + docs/reference/online-stores/README.md | 4 + docs/reference/online-stores/remote.md | 21 + infra/scripts/pixi/pixi.lock | 660 +++++++++++------- infra/scripts/pixi/pixi.toml | 10 +- sdk/python/feast/feature_store.py | 67 +- .../feast/infra/online_stores/remote.py | 174 +++++ .../feast/infra/online_stores/sqlite.py | 258 ++++++- sdk/python/feast/repo_config.py | 1 + .../requirements/py3.10-ci-requirements.txt | 58 +- .../requirements/py3.10-requirements.txt | 2 + .../requirements/py3.11-ci-requirements.txt | 83 +-- .../requirements/py3.11-requirements.txt | 8 +- .../requirements/py3.9-ci-requirements.txt | 79 ++- .../requirements/py3.9-requirements.txt | 7 +- sdk/python/tests/conftest.py | 18 +- .../example_repos/example_feature_repo_1.py | 23 +- .../feature_repos/repo_configuration.py | 6 +- .../online_store/test_remote_online_store.py | 222 ++++++ .../registration/test_universal_cli.py | 4 +- .../tests/unit/online_store/__init__.py | 0 .../online_store/test_online_retrieval.py | 145 +++- .../test_on_demand_python_transformation.py | 4 + setup.py | 6 +- 26 files changed, 1465 insertions(+), 421 deletions(-) create mode 100644 docs/reference/online-stores/remote.md create mode 100644 sdk/python/feast/infra/online_stores/remote.py create mode 100644 sdk/python/tests/integration/online_store/test_remote_online_store.py create mode 100644 sdk/python/tests/unit/online_store/__init__.py diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index aede0da23da..3081d418fcf 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -86,7 +86,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11" ] + python-version: [ "3.11" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/Makefile b/Makefile index f00dd00d981..c7fdc861000 100644 --- a/Makefile +++ b/Makefile @@ -70,12 +70,6 @@ lock-python-dependencies-all: pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt" pixi run --environment py311 --manifest-path infra/scripts/pixi/pixi.toml "uv pip compile --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt" -lock-python-dependencies-all: - pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.9-requirements.txt" - pixi run --environment py39 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt" - pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --output-file sdk/python/requirements/py3.10-requirements.txt" - pixi run --environment py310 --manifest-path infra/scripts/pixi/pixi.toml "python -m piptools compile -U --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt" - benchmark-python: IS_TEST=True python -m pytest --integration --benchmark --benchmark-autosave --benchmark-save-data sdk/python/tests diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 37d9b9cdf87..b9ce7f408a0 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -13,7 +13,9 @@ Below are supported vector databases and implemented features: | Elasticsearch | [x] | [x] | | Milvus | [ ] | [ ] | | Faiss | [ ] | [ ] | +| SQLite | [x] | [ ] | +Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. ## Example @@ -108,4 +110,20 @@ def print_online_features(features): print(key, " : ", value) print_online_features(features) +``` + +### Configuration +We offer two Online Store options for Vector Databases. PGVector and SQLite. + +#### Installation with SQLite +If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command: +```bash +PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ + LDFLAGS="-L/opt/homebrew/opt/sqlite/lib" \ + CPPFLAGS="-I/opt/homebrew/opt/sqlite/include" \ + pyenv install 3.10.14 +``` +And you can the Feast install package via: +```bash +pip install feast[sqlite_vec] ``` \ No newline at end of file diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 686e820f4e7..b5f4eb8de89 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -61,3 +61,7 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli {% content-ref url="scylladb.md" %} [scylladb.md](scylladb.md) {% endcontent-ref %} + +{% content-ref url="remote.md" %} +[remote.md](remote.md) +{% endcontent-ref %} diff --git a/docs/reference/online-stores/remote.md b/docs/reference/online-stores/remote.md new file mode 100644 index 00000000000..c560fa6f223 --- /dev/null +++ b/docs/reference/online-stores/remote.md @@ -0,0 +1,21 @@ +# Remote online store + +## Description + +This remote online store will let you interact with remote feature server. At this moment this only supports the read operation. You can use this online store and able retrieve online features `store.get_online_features` from remote feature server. + +## Examples + +The registry is pointing to registry of remote feature store. If it is not accessible then should be configured to use remote registry. + +{% code title="feature_store.yaml" %} +```yaml +project: my-local-project + registry: /remote/data/registry.db + provider: local + online_store: + path: http://localhost:6566 + type: remote + entity_key_serialization_version: 2 +``` +{% endcode %} \ No newline at end of file diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock index 65b761156e1..f1ce2d26585 100644 --- a/infra/scripts/pixi/pixi.lock +++ b/infra/scripts/pixi/pixi.lock @@ -1,6 +1,20 @@ -version: 4 +version: 5 environments: default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + py310: channels: - url: https://conda.anaconda.org/conda-forge/ packages: @@ -9,36 +23,40 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.3-hab00c5b_0_cpython.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda - py310: + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 + py311: channels: - url: https://conda.anaconda.org/conda-forge/ packages: @@ -47,34 +65,41 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-hc881cc4_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.0-hd590300_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py39: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -84,34 +109,39 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.9.19-h0755675_0_cpython.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda + osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 packages: - kind: conda name: _libgcc_mutex @@ -142,6 +172,19 @@ packages: license_family: BSD size: 23621 timestamp: 1650670423406 +- kind: conda + name: bzip2 + version: 1.0.8 + build: h93a5062_5 + build_number: 5 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda + sha256: bfa84296a638bea78a8bb29abc493ee95f2a0218775642474a840411b950fe5f + md5: 1bbc659ca658bfd49a481b5ef7a0f40f + license: bzip2-1.0.6 + license_family: BSD + size: 122325 + timestamp: 1699280294368 - kind: conda name: bzip2 version: 1.0.8 @@ -169,52 +212,16 @@ packages: size: 155432 timestamp: 1706843687645 - kind: conda - name: click - version: 8.1.7 - build: unix_pyh707e725_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda - sha256: f0016cbab6ac4138a429e28dbcb904a90305b34b3fe41a9b89d697c90401caec - md5: f3ad426304898027fc619827ff428eca - depends: - - __unix - - python >=3.8 - license: BSD-3-Clause - license_family: BSD - size: 84437 - timestamp: 1692311973840 -- kind: conda - name: colorama - version: 0.4.6 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2 - sha256: 2c1b2e9755ce3102bca8d69e8f26e4f087ece73f50418186aee7c74bef8e1698 - md5: 3faab06a954c2a04039983f2c4a50d99 - depends: - - python >=3.7 - license: BSD-3-Clause - license_family: BSD - size: 25170 - timestamp: 1666700778190 -- kind: conda - name: importlib-metadata - version: 7.1.0 - build: pyha770c72_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-7.1.0-pyha770c72_0.conda - sha256: cc2e7d1f7f01cede30feafc1118b7aefa244d0a12224513734e24165ae12ba49 - md5: 0896606848b2dc5cebdf111b6543aa04 - depends: - - python >=3.8 - - zipp >=0.5 - license: Apache-2.0 - license_family: APACHE - size: 27043 - timestamp: 1710971498183 + name: ca-certificates + version: 2024.2.2 + build: hf0a4a13_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + sha256: 49bc3439816ac72d0c0e0f144b8cc870fdcc4adec2e861407ec818d8116b2204 + md5: fb416a1795f18dcc5a038bc2dc54edf9 + license: ISC + size: 155725 + timestamp: 1706844034242 - kind: conda name: ld_impl_linux-64 version: '2.40' @@ -229,6 +236,34 @@ packages: license_family: GPL size: 704696 timestamp: 1674833944779 +- kind: conda + name: ld_impl_linux-64 + version: '2.40' + build: h55db66e_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda + sha256: ef969eee228cfb71e55146eaecc6af065f468cb0bc0a5239bc053b39db0b5f09 + md5: 10569984e7db886e4f1abc2b47ad79a1 + constrains: + - binutils_impl_linux-64 2.40 + license: GPL-3.0-only + license_family: GPL + size: 713322 + timestamp: 1713651222435 +- kind: conda + name: libcxx + version: 17.0.6 + build: h5f092b4_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + sha256: 119d3d9306f537d4c89dc99ed99b94c396d262f0b06f7833243646f68884f2c2 + md5: a96fd5dda8ce56c86a971e0fa02751d0 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + size: 1248885 + timestamp: 1715020154867 - kind: conda name: libexpat version: 2.6.2 @@ -245,6 +280,33 @@ packages: license_family: MIT size: 73730 timestamp: 1710362120304 +- kind: conda + name: libexpat + version: 2.6.2 + build: hebf3989_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda + sha256: ba7173ac30064ea901a4c9fb5a51846dcc25512ceb565759be7d18cbf3e5415e + md5: e3cde7cfa87f82f7cb13d482d5e0ad09 + constrains: + - expat 2.6.2.* + license: MIT + license_family: MIT + size: 63655 + timestamp: 1710362424980 +- kind: conda + name: libffi + version: 3.4.2 + build: h3422bc3_5 + build_number: 5 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 + sha256: 41b3d13efb775e340e4dba549ab5c029611ea6918703096b2eaa9c015c0750ca + md5: 086914b672be056eb70fd4285b6783b6 + license: MIT + license_family: MIT + size: 39020 + timestamp: 1636488587153 - kind: conda name: libffi version: 3.4.2 @@ -278,6 +340,24 @@ packages: license_family: GPL size: 770506 timestamp: 1706819192021 +- kind: conda + name: libgcc-ng + version: 13.2.0 + build: hc881cc4_6 + build_number: 6 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda + sha256: 836a0057525f1414de43642d357d0ab21ac7f85e24800b010dbc17d132e6efec + md5: df88796bd09a0d2ed292e59101478ad8 + depends: + - _libgcc_mutex 0.1 conda_forge + - _openmp_mutex >=4.5 + constrains: + - libgomp 13.2.0 hc881cc4_6 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 777315 + timestamp: 1713755001744 - kind: conda name: libgomp version: 13.2.0 @@ -293,6 +373,21 @@ packages: license_family: GPL size: 419751 timestamp: 1706819107383 +- kind: conda + name: libgomp + version: 13.2.0 + build: hc881cc4_6 + build_number: 6 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-hc881cc4_6.conda + sha256: e722b19b23b31a14b1592d5eceabb38dc52452ff5e4d346e330526971c22e52a + md5: aae89d3736661c36a5591788aebd0817 + depends: + - _libgcc_mutex 0.1 conda_forge + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 422363 + timestamp: 1713754915251 - kind: conda name: libnsl version: 2.0.1 @@ -307,6 +402,19 @@ packages: license_family: GPL size: 33408 timestamp: 1697359010159 +- kind: conda + name: libsqlite + version: 3.45.3 + build: h091b4b1_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + sha256: 4337f466eb55bbdc74e168b52ec8c38f598e3664244ec7a2536009036e2066cc + md5: c8c1186c7f3351f6ffddb97b1f54fc58 + depends: + - libzlib >=1.2.13,<2.0.0a0 + license: Unlicense + size: 824794 + timestamp: 1713367748819 - kind: conda name: libsqlite version: 3.45.3 @@ -321,6 +429,19 @@ packages: license: Unlicense size: 859858 timestamp: 1713367435849 +- kind: conda + name: libstdcxx-ng + version: 13.2.0 + build: h95c4c6d_6 + build_number: 6 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + sha256: 2616dbf9d28431eea20b6e307145c6a92ea0328a047c725ff34b0316de2617da + md5: 3cfab3e709f77e9f1b3d380eb622494a + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + size: 3842900 + timestamp: 1713755068572 - kind: conda name: libuuid version: 2.38.1 @@ -366,6 +487,23 @@ packages: license_family: Other size: 61588 timestamp: 1686575217516 +- kind: conda + name: libzlib + version: 1.2.13 + build: hfb2fe0b_6 + build_number: 6 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + sha256: 8b29a2386d99b8f58178951dcf19117b532cd9c4aa07623bf1667eae99755d32 + md5: 9c4e121cd926cab631bd1c4a61d18b17 + depends: + - __osx >=11.0 + constrains: + - zlib 1.2.13 *_6 + license: Zlib + license_family: Other + size: 46768 + timestamp: 1716874151980 - kind: conda name: ncurses version: 6.4.20240210 @@ -379,6 +517,17 @@ packages: license: X11 AND BSD-3-Clause size: 895669 timestamp: 1710866638986 +- kind: conda + name: ncurses + version: '6.5' + build: hb89a1cb_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda + sha256: 87d7cf716d9d930dab682cb57b3b8d3a61940b47d6703f3529a155c938a6990a + md5: b13ad5724ac9ae98b6b4fd87e4500ba4 + license: X11 AND BSD-3-Clause + size: 795131 + timestamp: 1715194898402 - kind: conda name: openssl version: 3.2.1 @@ -398,73 +547,40 @@ packages: size: 2865379 timestamp: 1710793235846 - kind: conda - name: packaging - version: '24.0' - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/packaging-24.0-pyhd8ed1ab_0.conda - sha256: a390182d74c31dfd713c16db888c92c277feeb6d1fe96ff9d9c105f9564be48a - md5: 248f521b64ce055e7feae3105e7abeb8 + name: openssl + version: 3.3.0 + build: hd590300_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.0-hd590300_0.conda + sha256: fdbf05e4db88c592366c90bb82e446edbe33c6e49e5130d51c580b2629c0b5d5 + md5: c0f3abb4a16477208bbd43a39bd56f18 depends: - - python >=3.8 + - ca-certificates + - libgcc-ng >=12 + constrains: + - pyopenssl >=22.1 license: Apache-2.0 - license_family: APACHE - size: 49832 - timestamp: 1710076089469 -- kind: conda - name: pip - version: '24.0' - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda - sha256: b7c1c5d8f13e8cb491c4bd1d0d1896a4cf80fc47de01059ad77509112b664a4a - md5: f586ac1e56c8638b64f9c8122a7b8a67 - depends: - - python >=3.7 - - setuptools - - wheel - license: MIT - license_family: MIT - size: 1398245 - timestamp: 1706960660581 -- kind: conda - name: pip-tools - version: 7.4.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/pip-tools-7.4.1-pyhd8ed1ab_0.conda - sha256: 5534c19a6233faed1c9109782322c9d31e536ce20448f8c90db3d864fb8f226d - md5: 73203bd783da9c37c2cdabb1f3b9d44d - depends: - - click >=7 - - pip >=21.2 - - python >=3.7 - - python-build - - setuptools - - wheel - license: BSD-3-Clause - license_family: BSD - size: 54113 - timestamp: 1709736180083 + license_family: Apache + size: 2895187 + timestamp: 1714466138265 - kind: conda - name: pyproject_hooks - version: 1.0.0 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/pyproject_hooks-1.0.0-pyhd8ed1ab_0.conda - sha256: 016340837fcfef57b351febcbe855eedf0c1f0ecfc910ed48c7fbd20535f9847 - md5: 21de50391d584eb7f4441b9de1ad773f + name: openssl + version: 3.3.0 + build: hfb2fe0b_3 + build_number: 3 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + sha256: 6f41c163ab57e7499dff092be4498614651f0f6432e12c2b9f06859a8bc39b75 + md5: 730f618b008b3c13c1e3f973408ddd67 depends: - - python >=3.7 - - tomli >=1.1.0 - license: MIT - license_family: MIT - size: 13867 - timestamp: 1670268791173 + - __osx >=11.0 + - ca-certificates + constrains: + - pyopenssl >=22.1 + license: Apache-2.0 + license_family: Apache + size: 2893954 + timestamp: 1716468329572 - kind: conda name: python version: 3.9.19 @@ -494,6 +610,54 @@ packages: license: Python-2.0 size: 23800555 timestamp: 1710940120866 +- kind: conda + name: python + version: 3.9.19 + build: hd7ebdb9_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + sha256: 3b93f7a405f334043758dfa8aaca050429a954a37721a6462ebd20e94ef7c5a0 + md5: 45c4d173b12154f746be3b49b1190634 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.9.* *_cp39 + license: Python-2.0 + size: 11847835 + timestamp: 1710939779164 +- kind: conda + name: python + version: 3.10.14 + build: h2469fbe_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + sha256: 454d609fe25daedce9e886efcbfcadad103ed0362e7cb6d2bcddec90b1ecd3ee + md5: 4ae999c8227c6d8c7623d32d51d25ea9 + depends: + - bzip2 >=1.0.8,<2.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.10.* *_cp310 + license: Python-2.0 + size: 12336005 + timestamp: 1710939659384 - kind: conda name: python version: 3.10.14 @@ -525,12 +689,38 @@ packages: timestamp: 1710939725109 - kind: conda name: python - version: 3.12.3 - build: hab00c5b_0_cpython + version: 3.11.9 + build: h932a869_0_cpython + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + sha256: a436ceabde1f056a0ac3e347dadc780ee2a135a421ddb6e9a469370769829e3c + md5: 293e0713ae804b5527a673e7605c04fc + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.6.2,<3.0a0 + - libffi >=3.4,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 + - ncurses >=6.4.20240210,<7.0a0 + - openssl >=3.2.1,<4.0a0 + - readline >=8.2,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + - xz >=5.2.6,<6.0a0 + constrains: + - python_abi 3.11.* *_cp311 + license: Python-2.0 + size: 14644189 + timestamp: 1713552154779 +- kind: conda + name: python + version: 3.11.9 + build: hb806964_0_cpython subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.3-hab00c5b_0_cpython.conda - sha256: f9865bcbff69f15fd89a33a2da12ad616e98d65ce7c83c644b92e66e5016b227 - md5: 2540b74d304f71d3e89c81209db4db84 + url: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda + sha256: 177f33a1fb8d3476b38f73c37b42f01c0b014fa0e039a701fd9f83d83aae6d40 + md5: ac68acfa8b558ed406c75e98d3428d7b depends: - bzip2 >=1.0.8,<2.0a0 - ld_impl_linux-64 >=2.36.1 @@ -538,7 +728,7 @@ packages: - libffi >=3.4,<4.0a0 - libgcc-ng >=12 - libnsl >=2.0.1,<2.1.0a0 - - libsqlite >=3.45.2,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 - libuuid >=2.38.1,<3.0a0 - libxcrypt >=4.4.36 - libzlib >=1.2.13,<1.3.0a0 @@ -549,32 +739,10 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.12.* *_cp312 + - python_abi 3.11.* *_cp311 license: Python-2.0 - size: 31991381 - timestamp: 1713208036041 -- kind: conda - name: python-build - version: 1.2.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/python-build-1.2.1-pyhd8ed1ab_0.conda - sha256: 3104051be7279d1b15f0a4be79f4bfeaf3a42b2900d24a7ad8e980df903fe8db - md5: d657cde3b3943fcedf6038138eea84de - depends: - - colorama - - importlib-metadata >=4.6 - - packaging >=19.0 - - pyproject_hooks - - python >=3.8 - - tomli >=1.1.0 - constrains: - - build <0 - license: MIT - license_family: MIT - size: 24434 - timestamp: 1711647439510 + size: 30884494 + timestamp: 1713553104915 - kind: conda name: readline version: '8.2' @@ -592,20 +760,35 @@ packages: size: 281456 timestamp: 1679532220005 - kind: conda - name: setuptools - version: 69.5.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/setuptools-69.5.1-pyhd8ed1ab_0.conda - sha256: 72d143408507043628b32bed089730b6d5f5445eccc44b59911ec9f262e365e7 - md5: 7462280d81f639363e6e63c81276bd9e + name: readline + version: '8.2' + build: h92ec313_1 + build_number: 1 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda + sha256: a1dfa679ac3f6007362386576a704ad2d0d7a02e98f5d0b115f207a2da63e884 + md5: 8cbb776a2f641b943d413b3e19df71f4 depends: - - python >=3.8 - license: MIT - license_family: MIT - size: 501790 - timestamp: 1713094963112 + - ncurses >=6.3,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 250351 + timestamp: 1679532511311 +- kind: conda + name: tk + version: 8.6.13 + build: h5083fa2_1 + build_number: 1 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda + sha256: 72457ad031b4c048e5891f3f6cb27a53cb479db68a52d965f796910e71a403a8 + md5: b50a57ba89c32b62428b71a875291c9b + depends: + - libzlib >=1.2.13,<1.3.0a0 + license: TCL + license_family: BSD + size: 3145523 + timestamp: 1699202432999 - kind: conda name: tk version: 8.6.13 @@ -622,21 +805,6 @@ packages: license_family: BSD size: 3318875 timestamp: 1699202167581 -- kind: conda - name: tomli - version: 2.0.1 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/tomli-2.0.1-pyhd8ed1ab_0.tar.bz2 - sha256: 4cd48aba7cd026d17e86886af48d0d2ebc67ed36f87f6534f4b67138f5a5a58f - md5: 5844808ffab9ebdb694585b50ba02a96 - depends: - - python >=3.7 - license: MIT - license_family: MIT - size: 15940 - timestamp: 1644342331069 - kind: conda name: tzdata version: 2024a @@ -650,21 +818,35 @@ packages: size: 119815 timestamp: 1706886945727 - kind: conda - name: wheel - version: 0.43.0 - build: pyhd8ed1ab_1 - build_number: 1 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda - sha256: cb318f066afd6fd64619f14c030569faf3f53e6f50abf743b4c865e7d95b96bc - md5: 0b5293a157c2b5cd513dd1b03d8d3aae + name: uv + version: 0.1.39 + build: h0ea3d13_0 + subdir: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + sha256: 763d149b6f4f5c70c91e4106d3a48409c48283ed2e27392578998fb2441f23d8 + md5: c3206e7ca254e50b3556917886f9b12b depends: - - python >=3.8 - license: MIT - license_family: MIT - size: 57963 - timestamp: 1711546009410 + - libgcc-ng >=12 + - libstdcxx-ng >=12 + license: Apache-2.0 OR MIT + size: 11891252 + timestamp: 1714233659570 +- kind: conda + name: uv + version: 0.1.45 + build: hc069d6b_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + sha256: 80dfc19f2ef473e86e718361847d1d598e95ffd0c0f5de7d07cda35d25f6aef5 + md5: 9192238a60bc6da9c41092990c31eb41 + depends: + - __osx >=11.0 + - libcxx >=16 + constrains: + - __osx >=11.0 + license: Apache-2.0 OR MIT + size: 9231858 + timestamp: 1716265232676 - kind: conda name: xz version: 5.2.6 @@ -679,17 +861,13 @@ packages: size: 418368 timestamp: 1660346797927 - kind: conda - name: zipp - version: 3.17.0 - build: pyhd8ed1ab_0 - subdir: noarch - noarch: python - url: https://conda.anaconda.org/conda-forge/noarch/zipp-3.17.0-pyhd8ed1ab_0.conda - sha256: bced1423fdbf77bca0a735187d05d9b9812d2163f60ab426fc10f11f92ecbe26 - md5: 2e4d6bc0b14e10f895fc6791a7d9b26a - depends: - - python >=3.8 - license: MIT - license_family: MIT - size: 18954 - timestamp: 1695255262261 + name: xz + version: 5.2.6 + build: h57fd34a_0 + subdir: osx-arm64 + url: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 + sha256: 59d78af0c3e071021cfe82dc40134c19dab8cdf804324b62940f5c8cd71803ec + md5: 39c6b54e94014701dd157f4f576ed211 + license: LGPL-2.1 and GPL-2.0 + size: 235693 + timestamp: 1660346961024 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index 80a29d3a59a..10179339f70 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -1,12 +1,12 @@ [project] name = "pixi-feast" channels = ["conda-forge"] -platforms = ["linux-64"] +platforms = ["linux-64", "osx-arm64"] [tasks] [dependencies] -pip-tools = ">=7.4.1,<7.5" +uv = ">=0.1.39,<0.2" [feature.py39.dependencies] python = "~=3.9.0" @@ -14,6 +14,10 @@ python = "~=3.9.0" [feature.py310.dependencies] python = "~=3.10.0" +[feature.py311.dependencies] +python = "~=3.11.0" + [environments] py39 = ["py39"] -py310 = ["py310"] \ No newline at end of file +py310 = ["py310"] +py311 = ["py311"] diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 8a0ebc6ddf9..577bd3fe52e 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -13,6 +13,7 @@ # limitations under the License. import copy import itertools +import logging import os import warnings from collections import Counter, defaultdict @@ -247,6 +248,20 @@ def list_feature_services(self) -> List[FeatureService]: """ return self._registry.list_feature_services(self.project) + def list_all_feature_views( + self, allow_cache: bool = False + ) -> List[Union[FeatureView, StreamFeatureView, OnDemandFeatureView]]: + """ + Retrieves the list of feature views from the registry. + + Args: + allow_cache: Whether to allow returning entities from a cached registry. + + Returns: + A list of feature views. + """ + return self._list_all_feature_views(allow_cache) + def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: """ Retrieves the list of feature views from the registry. @@ -257,12 +272,50 @@ def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: Returns: A list of feature views. """ + logging.warning( + "list_feature_views will make breaking changes. Please use list_batch_feature_views instead. " + "list_feature_views will behave like list_all_feature_views in the future." + ) return self._list_feature_views(allow_cache) + def _list_all_feature_views( + self, + allow_cache: bool = False, + ) -> List[Union[FeatureView, StreamFeatureView, OnDemandFeatureView]]: + all_feature_views = ( + self._list_feature_views(allow_cache) + + self._list_stream_feature_views(allow_cache) + + self.list_on_demand_feature_views(allow_cache) + ) + return all_feature_views + def _list_feature_views( self, allow_cache: bool = False, hide_dummy_entity: bool = True, + ) -> List[FeatureView]: + logging.warning( + "_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. " + "_list_feature_views will behave like _list_all_feature_views in the future." + ) + feature_views = [] + for fv in self._registry.list_feature_views( + self.project, allow_cache=allow_cache + ): + if ( + hide_dummy_entity + and fv.entities + and fv.entities[0] == DUMMY_ENTITY_NAME + ): + fv.entities = [] + fv.entity_columns = [] + feature_views.append(fv) + return feature_views + + def _list_batch_feature_views( + self, + allow_cache: bool = False, + hide_dummy_entity: bool = True, ) -> List[FeatureView]: feature_views = [] for fv in self._registry.list_feature_views( @@ -1881,18 +1934,28 @@ def _retrieve_online_documents( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) ( - requested_feature_views, + available_feature_views, _, ) = self._get_feature_views_to_use( features=[feature], allow_cache=True, hide_dummy_entity=False ) + requested_feature_view_name = ( + feature.split(":")[0] if isinstance(feature, str) else feature + ) + for feature_view in available_feature_views: + if feature_view.name == requested_feature_view_name: + requested_feature_view = feature_view + if not requested_feature_view: + raise ValueError( + f"Feature view {requested_feature_view} not found in the registry." + ) requested_feature = ( feature.split(":")[1] if isinstance(feature, str) else feature ) provider = self._get_provider() document_features = self._retrieve_from_online_store( provider, - requested_feature_views[0], + requested_feature_view, requested_feature, query, top_k, diff --git a/sdk/python/feast/infra/online_stores/remote.py b/sdk/python/feast/infra/online_stores/remote.py new file mode 100644 index 00000000000..a2dd35626dc --- /dev/null +++ b/sdk/python/feast/infra/online_stores/remote.py @@ -0,0 +1,174 @@ +# Copyright 2021 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +from datetime import datetime +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple + +import requests +from pydantic import StrictStr + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.online_stores.online_store import OnlineStore +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel +from feast.type_map import python_values_to_proto_values +from feast.value_type import ValueType + +logger = logging.getLogger(__name__) + + +class RemoteOnlineStoreConfig(FeastConfigBaseModel): + """Remote Online store config for remote online store""" + + type: Literal["remote"] = "remote" + """Online store type selector""" + + path: StrictStr = "http://localhost:6566" + """ str: Path to metadata store. + If type is 'remote', then this is a URL for registry server """ + + +class RemoteOnlineStore(OnlineStore): + """ + remote online store implementation wrapper to communicate with feast online server. + """ + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + pass + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + assert isinstance(config.online_store, RemoteOnlineStoreConfig) + config.online_store.__class__ = RemoteOnlineStoreConfig + + req_body = self._construct_online_read_api_json_request( + entity_keys, table, requested_features + ) + response = requests.post( + f"{config.online_store.path}/get-online-features", data=req_body + ) + if response.status_code == 200: + logger.debug("Able to retrieve the online features from feature server.") + response_json = json.loads(response.text) + event_ts = self._get_event_ts(response_json) + # Iterating over results and converting the API results in column format to row format. + result_tuples: List[ + Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]] + ] = [] + for feature_value_index in range(len(entity_keys)): + feature_values_dict: Dict[str, ValueProto] = dict() + for index, feature_name in enumerate( + response_json["metadata"]["feature_names"] + ): + if ( + requested_features is not None + and feature_name in requested_features + ): + if ( + response_json["results"][index]["statuses"][ + feature_value_index + ] + == "PRESENT" + ): + message = python_values_to_proto_values( + [ + response_json["results"][index]["values"][ + feature_value_index + ] + ], + ValueType.UNKNOWN, + ) + feature_values_dict[feature_name] = message[0] + else: + feature_values_dict[feature_name] = ValueProto() + + result_tuples.append((event_ts, feature_values_dict)) + return result_tuples + else: + error_msg = f"Unable to retrieve the online store data using feature server API. Error_code={response.status_code}, error_message={response.reason}" + logger.error(error_msg) + raise RuntimeError(error_msg) + + def _construct_online_read_api_json_request( + self, + entity_keys: List[EntityKeyProto], + table: FeatureView, + requested_features: Optional[List[str]] = None, + ): + api_requested_features = [] + if requested_features is not None: + for requested_feature in requested_features: + api_requested_features.append(f"{table.name}:{requested_feature}") + + entity_values = [] + entity_key = "" + for row in entity_keys: + entity_key = row.join_keys[0] + entity_values.append( + getattr(row.entity_values[0], row.entity_values[0].WhichOneof("val")) + ) + + req_body = json.dumps( + { + "features": api_requested_features, + "entities": {entity_key: entity_values}, + } + ) + return req_body + + def _check_if_feature_requested(self, feature_name, requested_features): + for requested_feature in requested_features: + if feature_name in requested_feature: + return True + return False + + def _get_event_ts(self, response_json) -> datetime: + event_ts = "" + if len(response_json["results"]) > 1: + event_ts = response_json["results"][1]["event_timestamps"][0] + return datetime.fromisoformat(event_ts.replace("Z", "+00:00")) + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + pass + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + pass diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 63d3ef03f51..41af14aaf16 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -14,10 +14,14 @@ import itertools import os import sqlite3 +import struct +import sys from datetime import datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +import sqlite_vec +from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr from feast import Entity @@ -29,6 +33,7 @@ from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.protos.feast.core.SqliteTable_pb2 import SqliteTable as SqliteTableProto from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig from feast.utils import to_naive_utc @@ -45,6 +50,12 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ + vec_enabled: Optional[bool] = False + """ (optional) Enable or disable sqlite-vss for vector search""" + + vector_len: Optional[int] = 512 + """ (optional) Length of the vector to be stored in the database""" + class SqliteOnlineStore(OnlineStore): """ @@ -73,6 +84,10 @@ def _get_conn(self, config: RepoConfig): if not self._conn: db_path = self._get_db_path(config) self._conn = _initialize_conn(db_path) + if sys.version_info[0:2] == (3, 10): + self._conn.enable_load_extension(True) # type: ignore + sqlite_vec.load(self._conn) + return self._conn def online_write_batch( @@ -80,7 +95,12 @@ def online_write_batch( config: RepoConfig, table: FeatureView, data: List[ - Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + Tuple[ + EntityKeyProto, + Dict[str, ValueProto], + datetime, + Optional[datetime], + ] ], progress: Optional[Callable[[int], Any]], ) -> None: @@ -98,36 +118,74 @@ def online_write_batch( if created_ts is not None: created_ts = to_naive_utc(created_ts) + table_name = _table_id(project, table) for feature_name, val in values.items(): - conn.execute( - f""" - UPDATE {_table_id(project, table)} - SET value = ?, event_ts = ?, created_ts = ? - WHERE (entity_key = ? AND feature_name = ?) - """, - ( - # SET - val.SerializeToString(), - timestamp, - created_ts, - # WHERE - entity_key_bin, - feature_name, - ), - ) - - conn.execute( - f"""INSERT OR IGNORE INTO {_table_id(project, table)} - (entity_key, feature_name, value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?)""", - ( - entity_key_bin, - feature_name, - val.SerializeToString(), - timestamp, - created_ts, - ), - ) + if config.online_store.vec_enabled: + vector_bin = serialize_f32( + val.float_list_val.val, config.online_store.vector_len + ) # type: ignore + conn.execute( + f""" + UPDATE {table_name} + SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, + ( + # SET + val.SerializeToString(), + vector_bin, + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + + conn.execute( + f"""INSERT OR IGNORE INTO {table_name} + (entity_key, feature_name, value, vector_value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + vector_bin, + timestamp, + created_ts, + ), + ) + + else: + conn.execute( + f""" + UPDATE {table_name} + SET value = ?, event_ts = ?, created_ts = ? + WHERE (entity_key = ? AND feature_name = ?) + """, + ( + # SET + val.SerializeToString(), + timestamp, + created_ts, + # WHERE + entity_key_bin, + feature_name, + ), + ) + + conn.execute( + f"""INSERT OR IGNORE INTO {table_name} + (entity_key, feature_name, value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?)""", + ( + entity_key_bin, + feature_name, + val.SerializeToString(), + timestamp, + created_ts, + ), + ) if progress: progress(1) @@ -195,7 +253,7 @@ def update( for table in tables_to_keep: conn.execute( - f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) conn.execute( f"CREATE INDEX IF NOT EXISTS {_table_id(project, table)}_ek ON {_table_id(project, table)} (entity_key);" @@ -232,6 +290,124 @@ def teardown( except FileNotFoundError: pass + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = None, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + """ + + Args: + config: Feast configuration object + table: FeatureView object as the table to search + requested_feature: The requested feature as the column to search + embedding: The query embedding to search for + top_k: The number of items to return + Returns: + List of tuples containing the event timestamp, the document feature, the vector value, and the distance + """ + project = config.project + + if not config.online_store.vec_enabled: + raise ValueError("sqlite-vss is not enabled in the online store config") + + conn = self._get_conn(config) + cur = conn.cursor() + + # Convert the embedding to a binary format instead of using SerializeToString() + query_embedding_bin = serialize_f32(embedding, config.online_store.vector_len) + table_name = _table_id(project, table) + + cur.execute( + f""" + CREATE VIRTUAL TABLE vec_example using vec0( + vector_value float[{config.online_store.vector_len}] + ); + """ + ) + + # Currently I can only insert the embedding value without crashing SQLite, will report a bug + cur.execute( + f""" + INSERT INTO vec_example(rowid, vector_value) + select rowid, vector_value from {table_name} + """ + ) + cur.execute( + """ + INSERT INTO vec_example(rowid, vector_value) + VALUES (?, ?) + """, + (0, query_embedding_bin), + ) + + # Have to join this with the {table_name} to get the feature name and entity_key + # Also the `top_k` doesn't appear to be working for some reason + cur.execute( + f""" + select + fv.entity_key, + f.vector_value, + fv.value, + f.distance, + fv.event_ts + from ( + select + rowid, + vector_value, + distance + from vec_example + where vector_value match ? + order by distance + limit ? + ) f + left join {table_name} fv + on f.rowid = fv.rowid + """, + (query_embedding_bin, top_k), + ) + + rows = cur.fetchall() + + result: List[ + Tuple[ + Optional[datetime], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ] = [] + + for entity_key, _, string_value, distance, event_ts in rows: + feature_value_proto = ValueProto() + feature_value_proto.ParseFromString(string_value if string_value else b"") + vector_value_proto = ValueProto( + float_list_val=FloatListProto(val=embedding) + ) + distance_value_proto = ValueProto(float_val=distance) + + result.append( + ( + event_ts, + feature_value_proto, + vector_value_proto, + distance_value_proto, + ) + ) + + return result + def _initialize_conn(db_path: str): Path(db_path).parent.mkdir(exist_ok=True) @@ -246,6 +422,19 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" +def serialize_f32( + vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int +) -> bytes: + """serializes a list of floats into a compact "raw bytes" format""" + return struct.pack(f"{vector_length}f", *vector) + + +def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: + """deserializes a list of floats from a compact "raw bytes" format""" + num_floats = vector_length // 4 # 4 bytes per float + return list(struct.unpack(f"{num_floats}f", byte_vector)) + + class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. @@ -292,8 +481,11 @@ def from_proto(sqlite_table_proto: SqliteTableProto) -> Any: ) def update(self): + if sys.version_info[0:2] == (3, 10): + self.conn.enable_load_extension(True) + sqlite_vec.load(self.conn) self.conn.execute( - f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 6ef81794bf8..b6a767e0f44 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -64,6 +64,7 @@ "hazelcast": "feast.infra.online_stores.contrib.hazelcast_online_store.hazelcast_online_store.HazelcastOnlineStore", "ikv": "feast.infra.online_stores.contrib.ikv_online_store.ikv.IKVOnlineStore", "elasticsearch": "feast.infra.online_stores.contrib.elasticsearch.ElasticSearchOnlineStore", + "remote": "feast.infra.online_stores.remote.RemoteOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index fd5d4631e56..d0da39aef42 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -53,7 +53,7 @@ azure-core==1.30.1 # azure-storage-blob azure-identity==1.16.0 # via feast (setup.py) -azure-storage-blob==12.19.1 +azure-storage-blob==12.20.0 # via feast (setup.py) babel==2.15.0 # via @@ -124,7 +124,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via @@ -161,17 +161,17 @@ distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 +docker==7.1.0 # via # feast (setup.py) # testcontainers docutils==0.19 # via sphinx -duckdb==0.10.2 +duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.0 +duckdb-engine==0.12.1 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch @@ -230,7 +230,7 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.128.0 +google-api-python-client==2.131.0 # via firebase-admin google-auth==2.29.0 # via @@ -279,11 +279,11 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.13 +great-expectations==0.18.15 # via feast (setup.py) grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.63.0 +grpcio==1.64.0 # via # feast (setup.py) # google-api-core @@ -313,7 +313,7 @@ h11==0.14.0 # uvicorn happybase==1.2.0 # via feast (setup.py) -hazelcast-python-client==5.3.0 +hazelcast-python-client==5.4.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) @@ -355,12 +355,12 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.4 # via jupyterlab -ipython==8.24.0 +ipython==8.25.0 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via great-expectations isodate==0.6.1 # via azure-storage-blob @@ -402,7 +402,7 @@ jsonschema[format-nongpl]==4.22.0 # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.2 # via # ipykernel # jupyter-server @@ -420,7 +420,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab @@ -429,15 +429,15 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.8 +jupyterlab==4.2.1 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -506,9 +506,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -notebook==7.1.3 +notebook==7.2.0 # via great-expectations notebook-shim==0.2.4 # via @@ -536,7 +536,6 @@ packaging==24.0 # build # dask # db-dtypes - # docker # duckdb-engine # google-cloud-bigquery # great-expectations @@ -594,7 +593,7 @@ pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.45 # via ipython proto-plus==1.23.0 # via @@ -822,19 +821,18 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.3 +ruff==0.4.6 # via feast (setup.py) s3transfer==0.10.1 # via boto3 -scipy==1.13.0 +scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==69.5.1 +setuptools==70.0.0 # via # grpcio-tools # kubernetes - # nodeenv # pip-tools shellingham==1.5.4 # via typer @@ -857,7 +855,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.0 +snowflake-connector-python[pandas]==3.10.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -887,11 +885,13 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework +sqlite-vec==0.0.1a10 + # via feast (setup.py) stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 +substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -918,7 +918,7 @@ tomli==2.0.1 # pip-tools # pytest # pytest-env -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 # via @@ -981,7 +981,7 @@ types-redis==4.6.0.20240425 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 # via # feast (setup.py) # types-cffi @@ -1064,7 +1064,7 @@ werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 # via diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 72124636b63..23bd94feb5c 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -166,6 +166,8 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) +sqlite-vec==0.0.1a10 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index bd0647a3fee..643e3715c60 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -36,10 +36,6 @@ asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab -async-timeout==4.0.3 - # via - # aiohttp - # redis atpublic==4.1.0 # via ibis-framework attrs==23.2.0 @@ -53,7 +49,7 @@ azure-core==1.30.1 # azure-storage-blob azure-identity==1.16.0 # via feast (setup.py) -azure-storage-blob==12.19.1 +azure-storage-blob==12.20.0 # via feast (setup.py) babel==2.15.0 # via @@ -124,7 +120,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via @@ -153,7 +149,7 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.3 +deltalake==0.17.4 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -161,17 +157,17 @@ distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 +docker==7.1.0 # via # feast (setup.py) # testcontainers docutils==0.19 # via sphinx -duckdb==0.10.2 +duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.0 +duckdb-engine==0.12.1 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch @@ -181,11 +177,6 @@ email-validator==2.1.1 # via fastapi entrypoints==0.4 # via altair -exceptiongroup==1.2.1 - # via - # anyio - # ipython - # pytest execnet==2.1.1 # via pytest-xdist executing==2.0.1 @@ -230,7 +221,7 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.128.0 +google-api-python-client==2.131.0 # via firebase-admin google-auth==2.29.0 # via @@ -279,11 +270,11 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.13 +great-expectations==0.18.15 # via feast (setup.py) grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.63.0 +grpcio==1.64.0 # via # feast (setup.py) # google-api-core @@ -313,7 +304,7 @@ h11==0.14.0 # uvicorn happybase==1.2.0 # via feast (setup.py) -hazelcast-python-client==5.3.0 +hazelcast-python-client==5.4.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) @@ -355,12 +346,12 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.4 # via jupyterlab -ipython==8.24.0 +ipython==8.25.0 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via great-expectations isodate==0.6.1 # via azure-storage-blob @@ -402,7 +393,7 @@ jsonschema[format-nongpl]==4.22.0 # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.2 # via # ipykernel # jupyter-server @@ -420,7 +411,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab @@ -429,15 +420,15 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.8 +jupyterlab==4.2.1 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -506,9 +497,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -notebook==7.1.3 +notebook==7.2.0 # via great-expectations notebook-shim==0.2.4 # via @@ -536,7 +527,6 @@ packaging==24.0 # build # dask # db-dtypes - # docker # duckdb-engine # google-cloud-bigquery # great-expectations @@ -594,7 +584,7 @@ pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.45 # via ipython proto-plus==1.23.0 # via @@ -775,7 +765,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.28 +regex==2024.5.15 # via feast (setup.py) requests==2.31.0 # via @@ -822,19 +812,18 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.3 +ruff==0.4.6 # via feast (setup.py) s3transfer==0.10.1 # via boto3 -scipy==1.13.0 +scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==69.5.1 +setuptools==70.0.0 # via # grpcio-tools # kubernetes - # nodeenv # pip-tools shellingham==1.5.4 # via typer @@ -857,7 +846,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.0 +snowflake-connector-python[pandas]==3.10.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -887,11 +876,13 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework +sqlite-vec==0.0.1a10 + # via feast (setup.py) stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 +substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -909,16 +900,7 @@ tinycss2==1.3.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pytest - # pytest-env -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 # via @@ -981,7 +963,7 @@ types-redis==4.6.0.20240425 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 # via # feast (setup.py) # types-cffi @@ -991,8 +973,6 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via - # anyio - # async-lru # azure-core # azure-storage-blob # fastapi @@ -1007,7 +987,6 @@ typing-extensions==4.11.0 # testcontainers # typeguard # typer - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 @@ -1064,7 +1043,7 @@ werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 # via diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index a381a6262b2..9698eea6dff 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -40,8 +40,6 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via fastapi -exceptiongroup==1.2.1 - # via anyio fastapi==0.111.0 # via # feast (setup.py) @@ -166,6 +164,8 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) +sqlite-vec==0.0.1a10 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -174,8 +174,6 @@ tenacity==8.3.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.1 - # via mypy toolz==0.12.1 # via # dask @@ -190,7 +188,6 @@ types-protobuf==5.26.0.20240422 # via mypy-protobuf typing-extensions==4.11.0 # via - # anyio # fastapi # mypy # pydantic @@ -198,7 +195,6 @@ typing-extensions==4.11.0 # sqlalchemy # typeguard # typer - # uvicorn tzdata==2024.1 # via pandas ujson==5.9.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 2456c852481..8aca7006961 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -53,7 +53,7 @@ azure-core==1.30.1 # azure-storage-blob azure-identity==1.16.0 # via feast (setup.py) -azure-storage-blob==12.19.1 +azure-storage-blob==12.20.0 # via feast (setup.py) babel==2.15.0 # via @@ -124,7 +124,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov cryptography==42.0.7 # via @@ -153,7 +153,7 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.17.3 +deltalake==0.17.4 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -161,17 +161,17 @@ distlib==0.3.8 # via virtualenv dnspython==2.6.1 # via email-validator -docker==7.0.0 +docker==7.1.0 # via # feast (setup.py) # testcontainers docutils==0.19 # via sphinx -duckdb==0.10.2 +duckdb==0.10.3 # via # duckdb-engine # ibis-framework -duckdb-engine==0.12.0 +duckdb-engine==0.12.1 # via ibis-framework elastic-transport==8.13.1 # via elasticsearch @@ -230,7 +230,7 @@ google-api-core[grpc]==2.19.0 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.128.0 +google-api-python-client==2.131.0 # via firebase-admin google-auth==2.29.0 # via @@ -279,11 +279,11 @@ googleapis-common-protos[grpc]==1.63.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.13 +great-expectations==0.18.15 # via feast (setup.py) grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.63.0 +grpcio==1.64.0 # via # feast (setup.py) # google-api-core @@ -313,7 +313,7 @@ h11==0.14.0 # uvicorn happybase==1.2.0 # via feast (setup.py) -hazelcast-python-client==5.3.0 +hazelcast-python-client==5.4.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) @@ -350,7 +350,16 @@ idna==3.7 imagesize==1.4.1 # via sphinx importlib-metadata==7.1.0 - # via dask + # via + # build + # dask + # jupyter-client + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # nbconvert + # sphinx + # typeguard iniconfig==2.0.0 # via pytest ipykernel==6.29.4 @@ -360,7 +369,7 @@ ipython==8.18.1 # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via great-expectations isodate==0.6.1 # via azure-storage-blob @@ -402,7 +411,7 @@ jsonschema[format-nongpl]==4.22.0 # nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.2 # via # ipykernel # jupyter-server @@ -420,7 +429,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab @@ -429,15 +438,15 @@ jupyter-server==2.14.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.8 +jupyterlab==4.2.1 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-server==2.27.1 +jupyterlab-server==2.27.2 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) @@ -506,9 +515,9 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -notebook==7.1.3 +notebook==7.2.0 # via great-expectations notebook-shim==0.2.4 # via @@ -536,7 +545,6 @@ packaging==24.0 # build # dask # db-dtypes - # docker # duckdb-engine # google-cloud-bigquery # great-expectations @@ -594,7 +602,7 @@ pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.20.0 # via jupyter-server -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.45 # via ipython proto-plus==1.23.0 # via @@ -775,7 +783,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.4.28 +regex==2024.5.15 # via feast (setup.py) requests==2.31.0 # via @@ -822,19 +830,20 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruff==0.4.3 +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +ruff==0.4.6 # via feast (setup.py) s3transfer==0.10.1 # via boto3 -scipy==1.13.0 +scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==69.5.1 +setuptools==70.0.0 # via # grpcio-tools # kubernetes - # nodeenv # pip-tools shellingham==1.5.4 # via typer @@ -857,7 +866,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.10.0 +snowflake-connector-python[pandas]==3.10.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -887,11 +896,13 @@ sqlalchemy-views==0.3.2 # via ibis-framework sqlglot==20.11.0 # via ibis-framework +sqlite-vec==0.0.1a10 + # via feast (setup.py) stack-data==0.6.3 # via ipython starlette==0.37.2 # via fastapi -substrait==0.17.0 +substrait==0.19.0 # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) @@ -918,7 +929,7 @@ tomli==2.0.1 # pip-tools # pytest # pytest-env -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python toolz==0.12.1 # via @@ -965,7 +976,7 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.1.0.20240425 +types-pymysql==1.1.0.20240524 # via feast (setup.py) types-pyopenssl==24.1.0.20240425 # via types-redis @@ -981,7 +992,7 @@ types-redis==4.6.0.20240425 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.5.0.20240423 +types-setuptools==70.0.0.20240524 # via # feast (setup.py) # types-cffi @@ -991,6 +1002,7 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.11.0 # via + # aioitertools # anyio # async-lru # azure-core @@ -998,11 +1010,13 @@ typing-extensions==4.11.0 # fastapi # great-expectations # ibis-framework + # ipython # mypy # pydantic # pydantic-core # snowflake-connector-python # sqlalchemy + # starlette # testcontainers # typeguard # typer @@ -1031,6 +1045,7 @@ urllib3==1.26.18 # requests # responses # rockset + # snowflake-connector-python # testcontainers uvicorn[standard]==0.29.0 # via @@ -1063,7 +1078,7 @@ werkzeug==3.0.3 # via moto wheel==0.43.0 # via pip-tools -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets wrapt==1.16.0 # via diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 72f422bfac8..579f39135e3 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -69,7 +69,9 @@ idna==3.7 # httpx # requests importlib-metadata==7.1.0 - # via dask + # via + # dask + # typeguard jinja2==3.1.4 # via # feast (setup.py) @@ -166,6 +168,8 @@ sniffio==1.3.1 # httpx sqlalchemy[mypy]==2.0.30 # via feast (setup.py) +sqlite-vec==0.0.1a10 + # via feast (setup.py) starlette==0.37.2 # via fastapi tabulate==0.9.0 @@ -196,6 +200,7 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy + # starlette # typeguard # typer # uvicorn diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 7c875fc9bde..03677dd1507 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -32,8 +32,8 @@ create_basic_driver_dataset, create_document_dataset, ) -from tests.integration.feature_repos.integration_test_repo_config import ( # noqa: E402 - IntegrationTestRepoConfig, +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, # noqa: E402 ) from tests.integration.feature_repos.repo_configuration import ( # noqa: E402 AVAILABLE_OFFLINE_STORES, @@ -45,8 +45,8 @@ construct_universal_feature_views, construct_universal_test_data, ) -from tests.integration.feature_repos.universal.data_sources.file import ( # noqa: E402 - FileDataSourceCreator, +from tests.integration.feature_repos.universal.data_sources.file import ( + FileDataSourceCreator, # noqa: E402 ) from tests.integration.feature_repos.universal.entities import ( # noqa: E402 customer, @@ -173,7 +173,15 @@ def simple_dataset_2() -> pd.DataFrame: def start_test_local_server(repo_path: str, port: int): fs = FeatureStore(repo_path) - fs.serve("localhost", port, no_access_log=True) + fs.serve( + host="localhost", + port=port, + no_access_log=True, + keep_alive_timeout=30, + type_="http", + workers=1, + registry_ttl_sec=30, + ) @pytest.fixture diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index fbf1fbb9b07..20a8ad7bd86 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -4,7 +4,7 @@ from feast import Entity, FeatureService, FeatureView, Field, FileSource, PushSource from feast.on_demand_feature_view import on_demand_feature_view -from feast.types import Float32, Int64, String +from feast.types import Array, Float32, Int64, String # Note that file source paths are not validated, so there doesn't actually need to be any data # at the paths for these file sources. Since these paths are effectively fake, this example @@ -32,6 +32,12 @@ batch_source=driver_locations_source, ) +rag_documents_source = FileSource( + name="rag_documents_source", + path="data/rag_documents.parquet", + timestamp_field="event_timestamp", +) + driver = Entity( name="driver", # The name is derived from this argument, not object name. join_keys=["driver_id"], @@ -43,6 +49,10 @@ join_keys=["customer_id"], ) +item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], +) driver_locations = FeatureView( name="driver_locations", @@ -101,6 +111,17 @@ tags={}, ) +document_embeddings = FeatureView( + name="document_embeddings", + entities=[item], + schema=[ + Field(name="Embeddings", dtype=Array(Float32)), + Field(name="item_id", dtype=String), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), +) + @on_demand_feature_view( sources=[customer_profile], diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 0545496f33a..4c463c3b0e7 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -134,9 +134,7 @@ AVAILABLE_ONLINE_STORES: Dict[ str, Tuple[Union[str, Dict[Any, Any]], Optional[Type[OnlineStoreCreator]]] -] = { - "sqlite": ({"type": "sqlite"}, None), -} +] = {"sqlite": ({"type": "sqlite"}, None)} # Only configure Cloud DWH if running full integration tests if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True": @@ -153,7 +151,6 @@ AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) - # Uncomment to test using private Rockset account. Currently not enabled as # there is no dedicated Rockset instance for CI testing and there is no # containerized version of Rockset. @@ -487,7 +484,6 @@ def construct_test_environment( "arn:aws:iam::402087665549:role/lambda_execution_role", ), ) - else: feature_server = LocalFeatureServerConfig( feature_logging=FeatureLoggingConfig(enabled=True) diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py new file mode 100644 index 00000000000..96ddec4c166 --- /dev/null +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -0,0 +1,222 @@ +import os +import tempfile +from datetime import datetime +from multiprocessing import Process +from textwrap import dedent + +import pytest + +from feast.feature_store import FeatureStore +from feast.wait import wait_retry_backoff +from tests.conftest import start_test_local_server +from tests.utils.cli_repo_creator import CliRunner +from tests.utils.http_server import check_port_open, free_port + + +@pytest.mark.integration +@pytest.mark.universal_online_stores +def test_remote_online_store_read(): + with tempfile.TemporaryDirectory() as remote_server_tmp_dir, tempfile.TemporaryDirectory() as remote_client_tmp_dir: + server_store, server_url, registry_path = ( + _create_server_store_spin_feature_server(temp_dir=remote_server_tmp_dir) + ) + assert None not in (server_store, server_url, registry_path) + client_store = _create_remote_client_feature_store( + temp_dir=remote_client_tmp_dir, + server_registry_path=str(registry_path), + feature_server_url=server_url, + ) + assert client_store is not None + _assert_non_existing_entity_feature_views_entity( + client_store=client_store, server_store=server_store + ) + _assert_existing_feature_views_entity( + client_store=client_store, server_store=server_store + ) + _assert_non_existing_feature_views( + client_store=client_store, server_store=server_store + ) + + +def _assert_non_existing_entity_feature_views_entity( + client_store: FeatureStore, server_store: FeatureStore +): + features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ] + + entity_rows = [{"driver_id": 1234}] + _assert_client_server_online_stores_are_matching( + client_store=client_store, + server_store=server_store, + features=features, + entity_rows=entity_rows, + ) + + +def _assert_non_existing_feature_views( + client_store: FeatureStore, server_store: FeatureStore +): + features = [ + "driver_hourly_stats1:conv_rate", + "driver_hourly_stats1:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ] + + entity_rows = [{"driver_id": 1001}, {"driver_id": 1002}] + + with pytest.raises( + Exception, match="Feature view driver_hourly_stats1 does not exist" + ): + client_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + with pytest.raises( + Exception, match="Feature view driver_hourly_stats1 does not exist" + ): + server_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + +def _assert_existing_feature_views_entity( + client_store: FeatureStore, server_store: FeatureStore +): + features = [ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ] + + entity_rows = [{"driver_id": 1001}, {"driver_id": 1002}] + _assert_client_server_online_stores_are_matching( + client_store=client_store, + server_store=server_store, + features=features, + entity_rows=entity_rows, + ) + + features = ["driver_hourly_stats:conv_rate"] + _assert_client_server_online_stores_are_matching( + client_store=client_store, + server_store=server_store, + features=features, + entity_rows=entity_rows, + ) + + +def _assert_client_server_online_stores_are_matching( + client_store: FeatureStore, server_store: FeatureStore, features, entity_rows +): + online_features_from_client = client_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + assert online_features_from_client is not None + + online_features_from_server = server_store.get_online_features( + features=features, entity_rows=entity_rows + ).to_dict() + + assert online_features_from_server is not None + assert online_features_from_client is not None + assert online_features_from_client == online_features_from_server + + +def _create_server_store_spin_feature_server(temp_dir): + feast_server_port = free_port() + store = _default_store(str(temp_dir), "REMOTE_ONLINE_SERVER_PROJECT") + server_url = next( + _start_feature_server( + repo_path=str(store.repo_path), server_port=feast_server_port + ) + ) + print(f"Server started successfully, {server_url}") + return store, server_url, os.path.join(store.repo_path, "data", "registry.db") + + +def _default_store(temp_dir, project_name): + runner = CliRunner() + result = runner.run(["init", project_name], cwd=temp_dir) + repo_path = os.path.join(temp_dir, project_name, "feature_repo") + assert result.returncode == 0 + + result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_dir) + assert result.returncode == 0 + + fs = FeatureStore(repo_path=repo_path) + fs.materialize_incremental( + end_date=datetime.utcnow(), feature_views=["driver_hourly_stats"] + ) + return fs + + +def _create_remote_client_feature_store( + temp_dir, server_registry_path: str, feature_server_url: str +): + project_name = "REMOTE_ONLINE_CLIENT_PROJECT" + runner = CliRunner() + result = runner.run(["init", project_name], cwd=temp_dir) + assert result.returncode == 0 + repo_path = os.path.join(temp_dir, project_name, "feature_repo") + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + ) + + result = runner.run(["--chdir", repo_path, "apply"], cwd=temp_dir) + assert result.returncode == 0 + + return FeatureStore(repo_path=repo_path) + + +def _overwrite_remote_client_feature_store_yaml( + repo_path: str, registry_path: str, feature_server_url: str +): + repo_config = os.path.join(repo_path, "feature_store.yaml") + with open(repo_config, "w") as repo_config: + repo_config.write( + dedent( + f""" + project: REMOTE_ONLINE_CLIENT_PROJECT + registry: {registry_path} + provider: local + online_store: + path: {feature_server_url} + type: remote + entity_key_serialization_version: 2 + """ + ) + ) + + +def _start_feature_server(repo_path: str, server_port: int): + feast_server_process = Process( + target=start_test_local_server, args=(repo_path, server_port) + ) + feast_server_process.start() + _time_out_sec: int = 45 + # Wait for server to start + wait_retry_backoff( + lambda: (None, check_port_open("localhost", server_port)), + timeout_secs=_time_out_sec, + timeout_msg=f"Unable to start the feast server in {_time_out_sec} seconds for remote online store type, port={server_port}", + ) + + yield f"http://localhost:{server_port}" + + if feast_server_process.is_alive(): + feast_server_process.kill() + + # wait server to free the port + wait_retry_backoff( + lambda: ( + None, + not check_port_open("localhost", server_port), + ), + timeout_secs=30, + ) diff --git a/sdk/python/tests/integration/registration/test_universal_cli.py b/sdk/python/tests/integration/registration/test_universal_cli.py index c16b26fee65..fc90108d787 100644 --- a/sdk/python/tests/integration/registration/test_universal_cli.py +++ b/sdk/python/tests/integration/registration/test_universal_cli.py @@ -74,13 +74,13 @@ def test_universal_cli(): cwd=repo_path, ) assertpy.assert_that(result.returncode).is_equal_to(0) - assertpy.assert_that(fs.list_feature_views()).is_length(4) + assertpy.assert_that(fs.list_feature_views()).is_length(5) result = runner.run( ["data-sources", "describe", "customer_profile_source"], cwd=repo_path, ) assertpy.assert_that(result.returncode).is_equal_to(0) - assertpy.assert_that(fs.list_data_sources()).is_length(4) + assertpy.assert_that(fs.list_data_sources()).is_length(5) # entity & feature view describe commands should fail when objects don't exist result = runner.run(["entities", "describe", "foo"], cwd=repo_path) diff --git a/sdk/python/tests/unit/online_store/__init__.py b/sdk/python/tests/unit/online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 5368b1e11cd..13b220fbb97 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -1,20 +1,26 @@ import os +import platform +import sqlite3 +import sys import time from datetime import datetime +import numpy as np import pandas as pd import pytest +import sqlite_vec from pandas.testing import assert_frame_equal from feast import FeatureStore, RepoConfig from feast.errors import FeatureViewNotFoundException from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig from tests.utils.cli_repo_creator import CliRunner, get_example_repo -def test_online() -> None: +def test_get_online_features() -> None: """ Test reading from the online store in local mode. """ @@ -415,3 +421,140 @@ def test_online_to_df(): ] expected_df = pd.DataFrame({k: reversed(v) for (k, v) in df_dict.items()}) assert_frame_equal(result_df[ordered_column], expected_df) + + +@pytest.mark.skipif( + sys.version_info[0:2] != (3, 10) or platform.system() != "Darwin", + reason="Only works on Python 3.10 and MacOS", +) +def test_sqlite_get_online_documents() -> None: + """ + Test retrieving documents from the online store in local mode. + """ + n = 10 # number of samples - note: we'll actually double it + vector_length = 8 + runner = CliRunner() + with runner.local_repo( + get_example_repo("example_feature_repo_1.py"), "file" + ) as store: + store.config.online_store.vec_enabled = True + store.config.online_store.vector_len = vector_length + # Write some data to two tables + document_embeddings_fv = store.get_feature_view(name="document_embeddings") + + provider = store._get_provider() + + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + ) + for i in range(n) + ] + data = [] + for item_key in item_keys: + data.append( + ( + item_key, + { + "Embeddings": ValueProto( + float_list_val=FloatListProto( + val=np.random.random( + vector_length, + ) + ) + ) + }, + datetime.utcnow(), + datetime.utcnow(), + ) + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, + ) + documents_df = pd.DataFrame( + { + "item_id": [str(i) for i in range(n)], + "Embeddings": [ + np.random.random( + vector_length, + ) + for i in range(n) + ], + "event_timestamp": [datetime.utcnow() for _ in range(n)], + } + ) + + store.write_to_online_store( + feature_view_name="document_embeddings", + df=documents_df, + ) + + document_table = store._provider._online_store._conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' and name like '%_document_embeddings';" + ).fetchall() + assert len(document_table) == 1 + document_table_name = document_table[0][0] + record_count = len( + store._provider._online_store._conn.execute( + f"select * from {document_table_name}" + ).fetchall() + ) + assert record_count == len(data) + documents_df.shape[0] + + query_embedding = np.random.random( + vector_length, + ) + result = store.retrieve_online_documents( + feature="document_embeddings:Embeddings", query=query_embedding, top_k=3 + ).to_dict() + + assert "Embeddings" in result + assert "distance" in result + assert len(result["distance"]) == 3 + + +@pytest.mark.skipif( + sys.version_info[0:2] != (3, 10) or platform.system() != "Darwin", + reason="Only works on Python 3.10 and MacOS", +) +def test_sqlite_vec_import() -> None: + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + sqlite_vec.load(db) + + db.execute(""" + create virtual table vec_examples using vec0( + sample_embedding float[8] + ); + """) + + db.execute(""" + insert into vec_examples(rowid, sample_embedding) + values + (1, '[-0.200, 0.250, 0.341, -0.211, 0.645, 0.935, -0.316, -0.924]'), + (2, '[0.443, -0.501, 0.355, -0.771, 0.707, -0.708, -0.185, 0.362]'), + (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), + (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]'); + """) + + sqlite_version, vec_version = db.execute( + "select sqlite_version(), vec_version()" + ).fetchone() + assert vec_version == "v0.0.1-alpha.10" + print(f"sqlite_version={sqlite_version}, vec_version={vec_version}") + + result = db.execute(""" + select + rowid, + distance + from vec_examples + where sample_embedding match '[0.890, 0.544, 0.825, 0.961, 0.358, 0.0196, 0.521, 0.175]' + order by distance + limit 2; + """).fetchall() + result = [(rowid, round(distance, 2)) for rowid, distance in result] + assert result == [(2, 2.39), (1, 2.39)] diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index ebe797ffdbf..72e9b53a101 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -159,6 +159,10 @@ def python_singleton_view(inputs: dict[str, Any]) -> dict[str, Any]: self.store.write_to_online_store( feature_view_name="driver_hourly_stats", df=driver_df ) + assert len(self.store.list_all_feature_views()) == 4 + assert len(self.store.list_feature_views()) == 1 + assert len(self.store.list_on_demand_feature_views()) == 3 + assert len(self.store.list_stream_feature_views()) == 0 def test_python_pandas_parity(self): entity_rows = [ diff --git a/setup.py b/setup.py index c0a66ec53cb..9b3d0e55e62 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,6 @@ "uvicorn[standard]>=0.14.0,<1", "gunicorn; platform_system != 'Windows'", "dask[dataframe]>=2024.4.2", - "bowler", # Needed for automatic repo upgrades ] GCP_REQUIRED = [ @@ -97,6 +96,9 @@ "pyspark>=3.0.0,<4", ] +SQLITE_VEC_REQUIRED = [ + "sqlite-vec==v0.0.1-alpha.10", +] TRINO_REQUIRED = ["trino>=0.305.0,<0.400.0", "regex"] POSTGRES_REQUIRED = [ @@ -215,6 +217,7 @@ + DUCKDB_REQUIRED + DELTA_REQUIRED + ELASTICSEARCH_REQUIRED + + SQLITE_VEC_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -382,6 +385,7 @@ def run(self): "ikv": IKV_REQUIRED, "delta": DELTA_REQUIRED, "elasticsearch": ELASTICSEARCH_REQUIRED, + "sqlite_vec": SQLITE_VEC_REQUIRED, }, include_package_data=True, license="Apache",