Skip to content

Commit

Permalink
feat: Add duckdb offline store (#3981)
Browse files Browse the repository at this point in the history
* add ibis and duckdb offline stores

Signed-off-by: tokoko <togurg14@freeuni.edu.ge>

* add linter ignore rule in ibis

Signed-off-by: tokoko <togurg14@freeuni.edu.ge>

* add linter ignore rule in ibis

Signed-off-by: tokoko <togurg14@freeuni.edu.ge>

---------

Signed-off-by: tokoko <togurg14@freeuni.edu.ge>
  • Loading branch information
tokoko authored Mar 5, 2024
1 parent 03dae13 commit 161547b
Show file tree
Hide file tree
Showing 8 changed files with 463 additions and 1 deletion.
13 changes: 13 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,19 @@ test-python-universal-athena:
not test_snowflake" \
sdk/python/tests

test-python-universal-duckdb:
PYTHONPATH='.' \
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.duckdb_repo_configuration \
FEAST_USAGE=False IS_TEST=True \
python -m pytest -n 8 --integration \
-k "not test_nullable_online_store and \
not gcs_registry and \
not s3_registry and \
not test_snowflake and \
not bigquery and \
not test_spark_materialization_consistency" \
sdk/python/tests

test-python-universal-postgres-offline:
PYTHONPATH='.' \
FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.postgres_repo_configuration \
Expand Down
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import ibis
from pydantic import StrictStr

from feast.infra.offline_stores.contrib.ibis_offline_store.ibis import IbisOfflineStore
from feast.repo_config import FeastConfigBaseModel


class DuckDBOfflineStoreConfig(FeastConfigBaseModel):
type: StrictStr = "duckdb"
# """ Offline store type selector"""


class DuckDBOfflineStore(IbisOfflineStore):
@staticmethod
def setup_ibis_backend():
# there's no need to call setup as duckdb is default ibis backend
ibis.set_backend("duckdb")
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
from feast.infra.offline_stores.contrib.duckdb_offline_store.duckdb import (
DuckDBOfflineStoreConfig,
)
from tests.integration.feature_repos.universal.data_sources.file import ( # noqa: E402
FileDataSourceCreator,
)


class DuckDBDataSourceCreator(FileDataSourceCreator):
def create_offline_store_config(self):
self.duckdb_offline_store_config = DuckDBOfflineStoreConfig()
return self.duckdb_offline_store_config


AVAILABLE_OFFLINE_STORES = [
("local", DuckDBDataSourceCreator),
]

AVAILABLE_ONLINE_STORES = {"sqlite": ({"type": "sqlite"}, None)}
Empty file.
Loading

0 comments on commit 161547b

Please sign in to comment.