From 5a5278d134ecea94474c316c1c06251b67285f8e Mon Sep 17 00:00:00 2001 From: tokoko Date: Mon, 12 Feb 2024 05:18:00 +0000 Subject: [PATCH 1/6] decouple transformation from odfvs Signed-off-by: tokoko --- protos/feast/core/OnDemandFeatureView.proto | 4 +- .../feast/infra/registry/base_registry.py | 2 +- sdk/python/feast/on_demand_feature_view.py | 69 +++++++++---------- .../feast/on_demand_pandas_transformation.py | 56 +++++++++++++++ .../feature_repos/universal/feature_views.py | 11 +-- .../tests/unit/test_on_demand_feature_view.py | 37 +++++++--- 6 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 sdk/python/feast/on_demand_pandas_transformation.py diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 50bf8b6f55..741d46e39e 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -48,7 +48,9 @@ message OnDemandFeatureViewSpec { // Map of sources for this feature view. map sources = 4; - UserDefinedFunction user_defined_function = 5; + oneof transformation { + UserDefinedFunction user_defined_function = 5; + } // Description of the on demand feature view. string description = 6; diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index f89b079478..e1caa083c7 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -629,7 +629,7 @@ def to_dict(self, project: str) -> Dict[str, List[Any]]: odfv_dict["spec"]["userDefinedFunction"][ "body" - ] = on_demand_feature_view.udf_string + ] = on_demand_feature_view.transformation.udf_string registry_dict["onDemandFeatureViews"].append(odfv_dict) for request_feature_view in sorted( self.list_request_feature_views(project=project), diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index fcafeaa2bc..41d7cb817a 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -2,7 +2,6 @@ import functools import warnings from datetime import datetime -from types import FunctionType from typing import Any, Dict, List, Optional, Type, Union import dill @@ -16,6 +15,7 @@ from feast.feature_view import FeatureView from feast.feature_view_projection import FeatureViewProjection from feast.field import Field, from_value_type +from feast.on_demand_pandas_transformation import OnDemandPandasTransformation from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( OnDemandFeatureView as OnDemandFeatureViewProto, ) @@ -24,9 +24,6 @@ OnDemandFeatureViewSpec, OnDemandSource, ) -from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( - UserDefinedFunction as UserDefinedFunctionProto, -) from feast.type_map import ( feast_value_type_to_pandas_type, python_type_to_feast_value_type, @@ -51,8 +48,7 @@ class OnDemandFeatureView(BaseFeatureView): sources with type FeatureViewProjection. source_request_sources: A map from input source names to the actual input sources with type RequestSource. - udf: The user defined transformation function, which must take pandas dataframes - as inputs. + transformation: The user defined transformation. description: A human-readable description. tags: A dictionary of key-value pairs to store arbitrary metadata. owner: The owner of the on demand feature view, typically the email of the primary @@ -63,8 +59,7 @@ class OnDemandFeatureView(BaseFeatureView): features: List[Field] source_feature_view_projections: Dict[str, FeatureViewProjection] source_request_sources: Dict[str, RequestSource] - udf: FunctionType - udf_string: str + transformation: Union[OnDemandPandasTransformation] description: str tags: Dict[str, str] owner: str @@ -82,8 +77,7 @@ def __init__( # noqa: C901 FeatureViewProjection, ] ], - udf: FunctionType, - udf_string: str = "", + transformation: Union[OnDemandPandasTransformation], description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", @@ -98,9 +92,7 @@ def __init__( # noqa: C901 sources: A map from input source names to the actual input sources, which may be feature views, or request data sources. These sources serve as inputs to the udf, which will refer to them by name. - udf: The user defined transformation function, which must take pandas - dataframes as inputs. - udf_string: The source code version of the udf (for diffing and displaying in Web UI) + transformation: The user defined transformation. description (optional): A human-readable description. tags (optional): A dictionary of key-value pairs to store arbitrary metadata. owner (optional): The owner of the on demand feature view, typically the email @@ -126,8 +118,7 @@ def __init__( # noqa: C901 odfv_source.name ] = odfv_source.projection - self.udf = udf # type: ignore - self.udf_string = udf_string + self.transformation = transformation @property def proto_class(self) -> Type[OnDemandFeatureViewProto]: @@ -139,8 +130,7 @@ def __copy__(self): schema=self.features, sources=list(self.source_feature_view_projections.values()) + list(self.source_request_sources.values()), - udf=self.udf, - udf_string=self.udf_string, + transformation=self.transformation, description=self.description, tags=self.tags, owner=self.owner, @@ -161,8 +151,7 @@ def __eq__(self, other): self.source_feature_view_projections != other.source_feature_view_projections or self.source_request_sources != other.source_request_sources - or self.udf_string != other.udf_string - or self.udf.__code__.co_code != other.udf.__code__.co_code + or self.transformation != other.transformation ): return False @@ -200,11 +189,9 @@ def to_proto(self) -> OnDemandFeatureViewProto: name=self.name, features=[feature.to_proto() for feature in self.features], sources=sources, - user_defined_function=UserDefinedFunctionProto( - name=self.udf.__name__, - body=dill.dumps(self.udf, recurse=True), - body_text=self.udf_string, - ), + user_defined_function=self.transformation.to_proto() + if type(self.transformation) == OnDemandPandasTransformation + else None, description=self.description, tags=self.tags, owner=self.owner, @@ -243,6 +230,16 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): RequestSource.from_proto(on_demand_source.request_data_source) ) + if ( + on_demand_feature_view_proto.spec.WhichOneof("transformation") + == "user_defined_function" + ): + transformation = OnDemandPandasTransformation.from_proto( + on_demand_feature_view_proto.spec.user_defined_function + ) + else: + raise Exception("At least one transformation type needs to be provided") + on_demand_feature_view_obj = cls( name=on_demand_feature_view_proto.spec.name, schema=[ @@ -253,10 +250,7 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): for feature in on_demand_feature_view_proto.spec.features ], sources=sources, - udf=dill.loads( - on_demand_feature_view_proto.spec.user_defined_function.body - ), - udf_string=on_demand_feature_view_proto.spec.user_defined_function.body_text, + transformation=transformation, description=on_demand_feature_view_proto.spec.description, tags=dict(on_demand_feature_view_proto.spec.tags), owner=on_demand_feature_view_proto.spec.owner, @@ -315,7 +309,8 @@ def get_transformed_features_df( columns_to_cleanup.append(full_feature_ref) # Compute transformed values and apply to each result row - df_with_transformed_features = self.udf.__call__(df_with_features) + + df_with_transformed_features = self.transformation.transform(df_with_features) # Work out whether the correct columns names are used. rename_columns: Dict[str, str] = {} @@ -335,7 +330,7 @@ def get_transformed_features_df( df_with_features.drop(columns=columns_to_cleanup, inplace=True) return df_with_transformed_features.rename(columns=rename_columns) - def infer_features(self): + def infer_features(self) -> None: """ Infers the set of features associated to this feature view from the input source. @@ -365,7 +360,7 @@ def infer_features(self): dtype = feast_value_type_to_pandas_type(field.dtype.to_value_type()) sample_val = rand_df_value[dtype] if dtype in rand_df_value else None df[f"{field.name}"] = pd.Series(sample_val, dtype=dtype) - output_df: pd.DataFrame = self.udf.__call__(df) + output_df: pd.DataFrame = self.transformation.transform(df) inferred_features = [] for f, dt in zip(output_df.columns, output_df.dtypes): inferred_features.append( @@ -396,7 +391,9 @@ def infer_features(self): ) @staticmethod - def get_requested_odfvs(feature_refs, project, registry): + def get_requested_odfvs( + feature_refs, project, registry + ) -> List["OnDemandFeatureView"]: all_on_demand_feature_views = registry.list_on_demand_feature_views( project, allow_cache=True ) @@ -438,7 +435,7 @@ def on_demand_feature_view( of the primary maintainer. """ - def mainify(obj): + def mainify(obj) -> None: # Needed to allow dill to properly serialize the udf. Otherwise, clients will need to have a file with the same # name as the original file defining the ODFV. if obj.__module__ != "__main__": @@ -447,15 +444,17 @@ def mainify(obj): def decorator(user_function): udf_string = dill.source.getsource(user_function) mainify(user_function) + + transformation = OnDemandPandasTransformation(user_function, udf_string) + on_demand_feature_view_obj = OnDemandFeatureView( name=user_function.__name__, sources=sources, schema=schema, - udf=user_function, + transformation=transformation, description=description, tags=tags, owner=owner, - udf_string=udf_string, ) functools.update_wrapper( wrapper=on_demand_feature_view_obj, wrapped=user_function diff --git a/sdk/python/feast/on_demand_pandas_transformation.py b/sdk/python/feast/on_demand_pandas_transformation.py new file mode 100644 index 0000000000..52d45893c5 --- /dev/null +++ b/sdk/python/feast/on_demand_pandas_transformation.py @@ -0,0 +1,56 @@ +from types import FunctionType + +import dill +import pandas as pd + +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + UserDefinedFunction as UserDefinedFunctionProto, +) + + +class OnDemandPandasTransformation: + def __init__(self, udf: FunctionType, udf_string: str = ""): + """ + Creates an OnDemandPandasTransformation object. + + Args: + udf: The user defined transformation function, which must take pandas + dataframes as inputs. + udf_string: The source code version of the udf (for diffing and displaying in Web UI) + """ + self.udf = udf + self.udf_string = udf_string + + def transform(self, df: pd.DataFrame) -> pd.DataFrame: + return self.udf.__call__(df) + + def __eq__(self, other): + if not isinstance(other, OnDemandPandasTransformation): + raise TypeError( + "Comparisons should only involve OnDemandPandasTransformation class objects." + ) + + if not super().__eq__(other): + return False + + if ( + self.udf_string != other.udf_string + or self.udf.__code__.co_code != other.udf.__code__.co_code + ): + return False + + return True + + def to_proto(self) -> UserDefinedFunctionProto: + return UserDefinedFunctionProto( + name=self.udf.__name__, + body=dill.dumps(self.udf, recurse=True), + body_text=self.udf_string, + ) + + @classmethod + def from_proto(cls, user_defined_function_proto: UserDefinedFunctionProto): + return OnDemandPandasTransformation( + udf=dill.loads(user_defined_function_proto.body), + udf_string=user_defined_function_proto.body_text, + ) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 5938a0c936..13a4806e56 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -14,6 +14,7 @@ StreamFeatureView, ) from feast.data_source import DataSource, RequestSource +from feast.on_demand_feature_view import OnDemandPandasTransformation from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 from tests.integration.feature_repos.universal.entities import ( customer, @@ -69,8 +70,9 @@ def conv_rate_plus_100_feature_view( name=conv_rate_plus_100.__name__, schema=[] if infer_features else _features, sources=sources, - udf=conv_rate_plus_100, - udf_string="raw udf source", + transformation=OnDemandPandasTransformation( + udf=conv_rate_plus_100, udf_string="raw udf source" + ), ) @@ -107,8 +109,9 @@ def similarity_feature_view( name=similarity.__name__, sources=sources, schema=[] if infer_features else _fields, - udf=similarity, - udf_string="similarity raw udf", + transformation=OnDemandPandasTransformation( + udf=similarity, udf_string="similarity raw udf" + ), ) diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index ca8e7b25cb..24a4af449b 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -17,7 +17,10 @@ from feast.feature_view import FeatureView from feast.field import Field from feast.infra.offline_stores.file_source import FileSource -from feast.on_demand_feature_view import OnDemandFeatureView +from feast.on_demand_feature_view import ( + OnDemandFeatureView, + OnDemandPandasTransformation, +) from feast.types import Float32 @@ -54,8 +57,9 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf1, - udf_string="udf1 source code", + transformation=OnDemandPandasTransformation( + udf=udf1, udf_string="udf1 source code" + ), ) on_demand_feature_view_2 = OnDemandFeatureView( name="my-on-demand-feature-view", @@ -64,8 +68,9 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf1, - udf_string="udf1 source code", + transformation=OnDemandPandasTransformation( + udf=udf1, udf_string="udf1 source code" + ), ) on_demand_feature_view_3 = OnDemandFeatureView( name="my-on-demand-feature-view", @@ -74,8 +79,9 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf2, - udf_string="udf2 source code", + transformation=OnDemandPandasTransformation( + udf=udf2, udf_string="udf2 source code" + ), ) on_demand_feature_view_4 = OnDemandFeatureView( name="my-on-demand-feature-view", @@ -84,8 +90,21 @@ def test_hash(): Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - udf=udf2, - udf_string="udf2 source code", + transformation=OnDemandPandasTransformation( + udf=udf2, udf_string="udf2 source code" + ), + description="test", + ) + on_demand_feature_view_4 = OnDemandFeatureView( + name="my-on-demand-feature-view", + sources=sources, + schema=[ + Field(name="output1", dtype=Float32), + Field(name="output2", dtype=Float32), + ], + transformation=OnDemandPandasTransformation( + udf=udf2, udf_string="udf2 source code" + ), description="test", ) From f0180e69e9a2bfbfa5152ce9ddbcddd8962a482a Mon Sep 17 00:00:00 2001 From: tokoko Date: Tue, 13 Feb 2024 20:52:22 +0000 Subject: [PATCH 2/6] OnDemandFeatureView: keep udf and udf_string parameters for backwards compatibility Signed-off-by: tokoko --- sdk/python/feast/on_demand_feature_view.py | 20 ++++++++++++++++++- .../tests/unit/test_on_demand_feature_view.py | 11 ++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 41d7cb817a..706f2ec4e4 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -2,6 +2,7 @@ import functools import warnings from datetime import datetime +from types import FunctionType from typing import Any, Dict, List, Optional, Type, Union import dill @@ -77,7 +78,9 @@ def __init__( # noqa: C901 FeatureViewProjection, ] ], - transformation: Union[OnDemandPandasTransformation], + udf: Optional[FunctionType] = None, + udf_string: str = "", + transformation: Optional[Union[OnDemandPandasTransformation]] = None, description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", @@ -92,6 +95,9 @@ def __init__( # noqa: C901 sources: A map from input source names to the actual input sources, which may be feature views, or request data sources. These sources serve as inputs to the udf, which will refer to them by name. + udf (deprecated): The user defined transformation function, which must take pandas + dataframes as inputs. + udf_string (deprecated): The source code version of the udf (for diffing and displaying in Web UI) transformation: The user defined transformation. description (optional): A human-readable description. tags (optional): A dictionary of key-value pairs to store arbitrary metadata. @@ -106,6 +112,18 @@ def __init__( # noqa: C901 owner=owner, ) + if not transformation: + if udf: + warnings.warn( + "udf and udf_string parameters are deprecated. Please use transformation=OnDemandPandasTransformation(udf, udf_string) instead.", + DeprecationWarning, + ) + transformation = OnDemandPandasTransformation(udf, udf_string) + else: + raise Exception( + "OnDemandFeatureView needs to be initialized with either transformation or udf arguments" + ) + self.source_feature_view_projections: Dict[str, FeatureViewProjection] = {} self.source_request_sources: Dict[str, RequestSource] = {} for odfv_source in sources: diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index 24a4af449b..721026ea46 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -95,16 +95,15 @@ def test_hash(): ), description="test", ) - on_demand_feature_view_4 = OnDemandFeatureView( + on_demand_feature_view_5 = OnDemandFeatureView( name="my-on-demand-feature-view", sources=sources, schema=[ Field(name="output1", dtype=Float32), Field(name="output2", dtype=Float32), ], - transformation=OnDemandPandasTransformation( - udf=udf2, udf_string="udf2 source code" - ), + udf=udf2, + udf_string="udf2 source code", description="test", ) @@ -124,3 +123,7 @@ def test_hash(): on_demand_feature_view_4, } assert len(s4) == 3 + + assert on_demand_feature_view_5.transformation == OnDemandPandasTransformation( + udf2, "udf2 source code" + ) From 9b936f100a2a752d27a003400245b821ace2e83d Mon Sep 17 00:00:00 2001 From: tokoko Date: Sat, 17 Feb 2024 19:17:25 +0000 Subject: [PATCH 3/6] fix linting issues Signed-off-by: tokoko --- .../tests/integration/feature_repos/universal/feature_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index a75e6cc2d0..421ef41601 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -14,8 +14,8 @@ StreamFeatureView, ) from feast.data_source import DataSource, RequestSource -from feast.on_demand_feature_view import OnDemandPandasTransformation from feast.feature_view_projection import FeatureViewProjection +from feast.on_demand_feature_view import OnDemandPandasTransformation from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 from tests.integration.feature_repos.universal.entities import ( customer, From 2f0c67cf44464ce5abd1cc8736114325cb5685ab Mon Sep 17 00:00:00 2001 From: tokoko Date: Sun, 18 Feb 2024 12:38:00 +0000 Subject: [PATCH 4/6] remove unused import in registry protos Signed-off-by: tokoko --- protos/feast/registry/RegistryServer.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/protos/feast/registry/RegistryServer.proto b/protos/feast/registry/RegistryServer.proto index 3e7773e89a..ab324f9bd1 100644 --- a/protos/feast/registry/RegistryServer.proto +++ b/protos/feast/registry/RegistryServer.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package feast.registry; -import "google/protobuf/timestamp.proto"; import "google/protobuf/empty.proto"; import "feast/core/Registry.proto"; import "feast/core/Entity.proto"; From 3cfc525eceec5ef2b5aeddd2986dc26b242e8d6d Mon Sep 17 00:00:00 2001 From: tokoko Date: Sat, 24 Feb 2024 17:04:07 +0000 Subject: [PATCH 5/6] feat: add OnDemandSubstraitTransformation Signed-off-by: tokoko --- protos/feast/core/OnDemandFeatureView.proto | 5 + sdk/python/feast/on_demand_feature_view.py | 55 +- .../on_demand_substrait_transformation.py | 50 + .../requirements/py3.11-ci-requirements.txt | 1000 ----------------- .../requirements/py3.11-requirements.txt | 234 ---- .../requirements/py3.8-ci-requirements.txt | 112 +- .../requirements/py3.8-requirements.txt | 33 +- .../requirements/py3.9-ci-requirements.txt | 117 +- .../requirements/py3.9-requirements.txt | 30 +- ...test_on_demand_substrait_transformation.py | 112 ++ setup.py | 9 +- 11 files changed, 413 insertions(+), 1344 deletions(-) create mode 100644 sdk/python/feast/on_demand_substrait_transformation.py delete mode 100644 sdk/python/requirements/py3.11-ci-requirements.txt delete mode 100644 sdk/python/requirements/py3.11-requirements.txt create mode 100644 sdk/python/tests/unit/test_on_demand_substrait_transformation.py diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 741d46e39e..c43b33c1d2 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -50,6 +50,7 @@ message OnDemandFeatureViewSpec { oneof transformation { UserDefinedFunction user_defined_function = 5; + OnDemandSubstraitTransformation on_demand_substrait_transformation = 9; } // Description of the on demand feature view. @@ -89,3 +90,7 @@ message UserDefinedFunction { // The string representation of the udf string body_text = 3; } + +message OnDemandSubstraitTransformation { + bytes substrait_plan = 1; +} \ No newline at end of file diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 706f2ec4e4..586286a3d4 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,5 +1,6 @@ import copy import functools +import inspect import warnings from datetime import datetime from types import FunctionType @@ -17,6 +18,7 @@ from feast.feature_view_projection import FeatureViewProjection from feast.field import Field, from_value_type from feast.on_demand_pandas_transformation import OnDemandPandasTransformation +from feast.on_demand_substrait_transformation import OnDemandSubstraitTransformation from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( OnDemandFeatureView as OnDemandFeatureViewProto, ) @@ -210,6 +212,9 @@ def to_proto(self) -> OnDemandFeatureViewProto: user_defined_function=self.transformation.to_proto() if type(self.transformation) == OnDemandPandasTransformation else None, + on_demand_substrait_transformation=self.transformation.to_proto() # type: ignore + if type(self.transformation) == OnDemandSubstraitTransformation + else None, description=self.description, tags=self.tags, owner=self.owner, @@ -255,6 +260,13 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): transformation = OnDemandPandasTransformation.from_proto( on_demand_feature_view_proto.spec.user_defined_function ) + elif ( + on_demand_feature_view_proto.spec.WhichOneof("transformation") + == "on_demand_substrait_transformation" + ): + transformation = OnDemandSubstraitTransformation.from_proto( + on_demand_feature_view_proto.spec.on_demand_substrait_transformation + ) else: raise Exception("At least one transformation type needs to be provided") @@ -460,10 +472,47 @@ def mainify(obj) -> None: obj.__module__ = "__main__" def decorator(user_function): - udf_string = dill.source.getsource(user_function) - mainify(user_function) + return_annotation = inspect.signature(user_function).return_annotation + if ( + return_annotation + and return_annotation.__module__ == "ibis.expr.types.relations" + and return_annotation.__name__ == "Table" + ): + import ibis + import ibis.expr.datatypes as dt + from ibis_substrait.compiler.core import SubstraitCompiler + + compiler = SubstraitCompiler() + + input_fields: Field = [] + + for s in sources: + if type(s) == FeatureView: + fields = s.projection.features + else: + fields = s.features + + input_fields.extend( + [ + ( + f.name, + dt.dtype( + feast_value_type_to_pandas_type(f.dtype.to_value_type()) + ), + ) + for f in fields + ] + ) + + expr = user_function(ibis.table(input_fields, "t")) - transformation = OnDemandPandasTransformation(user_function, udf_string) + transformation = OnDemandSubstraitTransformation( + substrait_plan=compiler.compile(expr).SerializeToString() + ) + else: + udf_string = dill.source.getsource(user_function) + mainify(user_function) + transformation = OnDemandPandasTransformation(user_function, udf_string) on_demand_feature_view_obj = OnDemandFeatureView( name=user_function.__name__, diff --git a/sdk/python/feast/on_demand_substrait_transformation.py b/sdk/python/feast/on_demand_substrait_transformation.py new file mode 100644 index 0000000000..4e92e77dc8 --- /dev/null +++ b/sdk/python/feast/on_demand_substrait_transformation.py @@ -0,0 +1,50 @@ +import pandas as pd +import pyarrow +import pyarrow.substrait as substrait # type: ignore # noqa + +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( + OnDemandSubstraitTransformation as OnDemandSubstraitTransformationProto, +) + + +class OnDemandSubstraitTransformation: + def __init__(self, substrait_plan: bytes): + """ + Creates an OnDemandSubstraitTransformation object. + + Args: + substrait_plan: The user-provided substrait plan. + """ + self.substrait_plan = substrait_plan + + def transform(self, df: pd.DataFrame) -> pd.DataFrame: + def table_provider(names, schema: pyarrow.Schema): + return pyarrow.Table.from_pandas(df[schema.names]) + + table: pyarrow.Table = pyarrow.substrait.run_query( + self.substrait_plan, table_provider=table_provider + ).read_all() + return table.to_pandas() + + def __eq__(self, other): + if not isinstance(other, OnDemandSubstraitTransformation): + raise TypeError( + "Comparisons should only involve OnDemandSubstraitTransformation class objects." + ) + + if not super().__eq__(other): + return False + + return self.substrait_plan == other.substrait_plan + + def to_proto(self) -> OnDemandSubstraitTransformationProto: + return OnDemandSubstraitTransformationProto(substrait_plan=self.substrait_plan) + + @classmethod + def from_proto( + cls, + on_demand_substrait_transformation_proto: OnDemandSubstraitTransformationProto, + ): + return OnDemandSubstraitTransformation( + substrait_plan=on_demand_substrait_transformation_proto.substrait_plan + ) diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt deleted file mode 100644 index bc9ddbe8e7..0000000000 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ /dev/null @@ -1,1000 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt -# -alabaster==0.7.16 - # via sphinx -altair==4.2.2 - # via great-expectations -annotated-types==0.6.0 - # via pydantic -anyio==4.2.0 - # via - # httpx - # jupyter-server - # starlette - # watchfiles -appdirs==1.4.4 - # via fissix -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.3.0 - # via isoduration -asn1crypto==1.5.1 - # via snowflake-connector-python -assertpy==1.1 - # via feast (setup.py) -asttokens==2.4.1 - # via stack-data -async-lru==2.0.4 - # via jupyterlab -async-timeout==4.0.3 - # via redis -attrs==23.2.0 - # via - # bowler - # jsonschema - # referencing -avro==1.10.0 - # via feast (setup.py) -azure-core==1.30.0 - # via - # azure-identity - # azure-storage-blob -azure-identity==1.15.0 - # via feast (setup.py) -azure-storage-blob==12.19.0 - # via feast (setup.py) -babel==2.14.0 - # via - # jupyterlab-server - # sphinx -beautifulsoup4==4.12.3 - # via nbconvert -black==22.12.0 - # via feast (setup.py) -bleach==6.1.0 - # via nbconvert -boto3==1.34.42 - # via - # feast (setup.py) - # moto -botocore==1.34.42 - # via - # boto3 - # moto - # s3transfer -bowler==0.9.0 - # via feast (setup.py) -build==1.0.3 - # via - # feast (setup.py) - # pip-tools -bytewax==0.18.2 - # via feast (setup.py) -cachecontrol==0.14.0 - # via firebase-admin -cachetools==5.3.2 - # via google-auth -cassandra-driver==3.29.0 - # via feast (setup.py) -certifi==2024.2.2 - # via - # httpcore - # httpx - # kubernetes - # minio - # requests - # snowflake-connector-python -cffi==1.16.0 - # via - # argon2-cffi-bindings - # cryptography - # snowflake-connector-python -cfgv==3.4.0 - # via pre-commit -charset-normalizer==3.3.2 - # via - # requests - # snowflake-connector-python -click==8.1.7 - # via - # black - # bowler - # dask - # feast (setup.py) - # geomet - # great-expectations - # moreorless - # pip-tools - # uvicorn -cloudpickle==3.0.0 - # via dask -colorama==0.4.6 - # via - # feast (setup.py) - # great-expectations -comm==0.2.1 - # via - # ipykernel - # ipywidgets -coverage[toml]==7.4.1 - # via pytest-cov -cryptography==41.0.7 - # via - # azure-identity - # azure-storage-blob - # feast (setup.py) - # great-expectations - # moto - # msal - # pyjwt - # pyopenssl - # snowflake-connector-python - # types-pyopenssl - # types-redis -dask==2024.2.0 - # via feast (setup.py) -db-dtypes==1.2.0 - # via google-cloud-bigquery -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -defusedxml==0.7.1 - # via nbconvert -deprecation==2.1.0 - # via testcontainers -dill==0.3.8 - # via - # bytewax - # feast (setup.py) - # multiprocess -distlib==0.3.8 - # via virtualenv -docker==7.0.0 - # via - # feast (setup.py) - # testcontainers -docutils==0.19 - # via sphinx -entrypoints==0.4 - # via altair -exceptiongroup==1.2.0 - # via - # anyio - # ipython - # pytest -execnet==2.0.2 - # via pytest-xdist -executing==2.0.1 - # via stack-data -fastapi==0.109.2 - # via feast (setup.py) -fastavro==1.9.4 - # via - # feast (setup.py) - # pandavro -fastjsonschema==2.19.1 - # via nbformat -filelock==3.13.1 - # via - # snowflake-connector-python - # virtualenv -firebase-admin==5.4.0 - # via feast (setup.py) -fissix==21.11.13 - # via bowler -flake8==6.0.0 - # via feast (setup.py) -fqdn==1.5.1 - # via jsonschema -fsspec==2023.12.2 - # via - # dask - # feast (setup.py) -geojson==2.5.0 - # via rockset -geomet==0.2.1.post1 - # via cassandra-driver -google-api-core[grpc]==2.17.1 - # via - # feast (setup.py) - # firebase-admin - # google-api-python-client - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-core - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-api-python-client==2.118.0 - # via firebase-admin -google-auth==2.27.0 - # via - # google-api-core - # google-api-python-client - # google-auth-httplib2 - # google-cloud-core - # google-cloud-storage - # kubernetes -google-auth-httplib2==0.2.0 - # via google-api-python-client -google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) -google-cloud-bigquery-storage==2.24.0 - # via feast (setup.py) -google-cloud-bigtable==2.23.0 - # via feast (setup.py) -google-cloud-core==2.4.1 - # via - # google-cloud-bigquery - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # google-cloud-storage -google-cloud-datastore==2.19.0 - # via feast (setup.py) -google-cloud-firestore==2.14.0 - # via firebase-admin -google-cloud-storage==2.14.0 - # via - # feast (setup.py) - # firebase-admin -google-crc32c==1.5.0 - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.7.0 - # via - # google-cloud-bigquery - # google-cloud-storage -googleapis-common-protos[grpc]==1.62.0 - # via - # feast (setup.py) - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -great-expectations==0.18.8 - # via feast (setup.py) -greenlet==3.0.3 - # via sqlalchemy -grpc-google-iam-v1==0.13.0 - # via google-cloud-bigtable -grpcio==1.60.1 - # via - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools -grpcio-health-checking==1.60.1 - # via feast (setup.py) -grpcio-reflection==1.60.1 - # via feast (setup.py) -grpcio-status==1.60.1 - # via google-api-core -grpcio-testing==1.60.1 - # via feast (setup.py) -grpcio-tools==1.60.1 - # via feast (setup.py) -gunicorn==21.2.0 - # via feast (setup.py) -h11==0.14.0 - # via - # httpcore - # uvicorn -happybase==1.2.0 - # via feast (setup.py) -hazelcast-python-client==5.3.0 - # via feast (setup.py) -hiredis==2.3.2 - # via feast (setup.py) -httpcore==1.0.3 - # via httpx -httplib2==0.22.0 - # via - # google-api-python-client - # google-auth-httplib2 -httptools==0.6.1 - # via uvicorn -httpx==0.26.0 - # via - # feast (setup.py) - # jupyterlab -identify==2.5.34 - # via pre-commit -idna==3.6 - # via - # anyio - # httpx - # jsonschema - # requests - # snowflake-connector-python -imagesize==1.4.1 - # via sphinx -importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.1.1 - # via feast (setup.py) -iniconfig==2.0.0 - # via pytest -ipykernel==6.29.2 - # via jupyterlab -ipython==8.21.0 - # via - # great-expectations - # ipykernel - # ipywidgets -ipywidgets==8.1.2 - # via great-expectations -isodate==0.6.1 - # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -isort==5.13.2 - # via feast (setup.py) -jedi==0.19.1 - # via ipython -jinja2==3.1.3 - # via - # altair - # feast (setup.py) - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # moto - # nbconvert - # sphinx -jmespath==1.0.1 - # via - # boto3 - # botocore -json5==0.9.14 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.21.1 - # via - # altair - # feast (setup.py) - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema -jupyter-client==8.6.0 - # via - # ipykernel - # jupyter-server - # nbclient -jupyter-core==5.7.1 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat -jupyter-events==0.9.0 - # via jupyter-server -jupyter-lsp==2.2.2 - # via jupyterlab -jupyter-server==2.12.5 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.5.2 - # via jupyter-server -jupyterlab==4.1.1 - # via notebook -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-server==2.25.3 - # via - # jupyterlab - # notebook -jupyterlab-widgets==3.0.10 - # via ipywidgets -kubernetes==20.13.0 - # via feast (setup.py) -locket==1.0.0 - # via partd -makefun==1.15.2 - # via great-expectations -markupsafe==2.1.5 - # via - # jinja2 - # nbconvert - # werkzeug -marshmallow==3.20.2 - # via great-expectations -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython -mccabe==0.7.0 - # via flake8 -minio==7.1.0 - # via feast (setup.py) -mistune==3.0.2 - # via - # great-expectations - # nbconvert -mmh3==4.1.0 - # via feast (setup.py) -mock==2.0.0 - # via feast (setup.py) -moreorless==0.4.0 - # via bowler -moto==4.2.14 - # via feast (setup.py) -msal==1.26.0 - # via - # azure-identity - # msal-extensions -msal-extensions==1.1.0 - # via azure-identity -msgpack==1.0.7 - # via cachecontrol -multiprocess==0.70.16 - # via bytewax -mypy==1.8.0 - # via - # feast (setup.py) - # sqlalchemy -mypy-extensions==1.0.0 - # via - # black - # mypy -mypy-protobuf==3.1.0 - # via feast (setup.py) -nbclient==0.9.0 - # via nbconvert -nbconvert==7.16.0 - # via jupyter-server -nbformat==5.9.2 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert -nest-asyncio==1.6.0 - # via ipykernel -nodeenv==1.8.0 - # via pre-commit -notebook==7.1.0 - # via great-expectations -notebook-shim==0.2.3 - # via - # jupyterlab - # notebook -numpy==1.24.4 - # via - # altair - # db-dtypes - # feast (setup.py) - # great-expectations - # pandas - # pandavro - # pyarrow - # scipy -oauthlib==3.2.2 - # via requests-oauthlib -overrides==7.7.0 - # via jupyter-server -packaging==23.2 - # via - # build - # dask - # db-dtypes - # deprecation - # docker - # google-cloud-bigquery - # great-expectations - # gunicorn - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server - # marshmallow - # msal-extensions - # nbconvert - # pytest - # snowflake-connector-python - # sphinx -pandas==1.5.3 - # via - # altair - # db-dtypes - # feast (setup.py) - # google-cloud-bigquery - # great-expectations - # pandavro - # snowflake-connector-python -pandavro==1.5.2 - # via feast (setup.py) -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.3 - # via jedi -partd==1.4.1 - # via dask -pathspec==0.12.1 - # via black -pbr==6.0.0 - # via mock -pexpect==4.9.0 - # via ipython -pip-tools==7.3.0 - # via feast (setup.py) -platformdirs==3.11.0 - # via - # black - # jupyter-core - # snowflake-connector-python - # virtualenv -pluggy==1.4.0 - # via pytest -ply==3.11 - # via thriftpy2 -portalocker==2.8.2 - # via msal-extensions -pre-commit==3.3.1 - # via feast (setup.py) -prometheus-client==0.20.0 - # via jupyter-server -prompt-toolkit==3.0.43 - # via ipython -proto-plus==1.23.0 - # via - # feast (setup.py) - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore -protobuf==4.23.3 - # via - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # google-cloud-bigquery-storage - # google-cloud-bigtable - # google-cloud-datastore - # google-cloud-firestore - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-health-checking - # grpcio-reflection - # grpcio-status - # grpcio-testing - # grpcio-tools - # mypy-protobuf - # proto-plus -psutil==5.9.0 - # via - # feast (setup.py) - # ipykernel -psycopg2-binary==2.9.9 - # via feast (setup.py) -ptyprocess==0.7.0 - # via - # pexpect - # terminado -pure-eval==0.2.2 - # via stack-data -py==1.11.0 - # via feast (setup.py) -py-cpuinfo==9.0.0 - # via pytest-benchmark -py4j==0.10.9.7 - # via pyspark -pyarrow==15.0.0 - # via - # db-dtypes - # feast (setup.py) - # google-cloud-bigquery - # snowflake-connector-python -pyasn1==0.5.1 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.3.0 - # via google-auth -pybindgen==0.22.1 - # via feast (setup.py) -pycodestyle==2.10.0 - # via flake8 -pycparser==2.21 - # via cffi -pydantic==2.6.1 - # via - # fastapi - # feast (setup.py) - # great-expectations -pydantic-core==2.16.2 - # via pydantic -pyflakes==3.0.1 - # via flake8 -pygments==2.17.2 - # via - # feast (setup.py) - # ipython - # nbconvert - # sphinx -pyjwt[crypto]==2.8.0 - # via - # msal - # snowflake-connector-python -pymssql==2.2.11 - # via feast (setup.py) -pymysql==1.1.0 - # via feast (setup.py) -pyodbc==5.1.0 - # via feast (setup.py) -pyopenssl==23.3.0 - # via snowflake-connector-python -pyparsing==3.1.1 - # via - # great-expectations - # httplib2 -pyproject-hooks==1.0.0 - # via build -pyspark==3.5.0 - # via feast (setup.py) -pytest==7.4.4 - # via - # feast (setup.py) - # pytest-benchmark - # pytest-cov - # pytest-lazy-fixture - # pytest-mock - # pytest-ordering - # pytest-timeout - # pytest-xdist -pytest-benchmark==3.4.1 - # via feast (setup.py) -pytest-cov==4.1.0 - # via feast (setup.py) -pytest-lazy-fixture==0.6.3 - # via feast (setup.py) -pytest-mock==1.10.4 - # via feast (setup.py) -pytest-ordering==0.6 - # via feast (setup.py) -pytest-timeout==1.4.2 - # via feast (setup.py) -pytest-xdist==3.5.0 - # via feast (setup.py) -python-dateutil==2.8.2 - # via - # arrow - # botocore - # google-cloud-bigquery - # great-expectations - # jupyter-client - # kubernetes - # moto - # pandas - # rockset - # trino -python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 - # via jupyter-events -pytz==2024.1 - # via - # great-expectations - # pandas - # snowflake-connector-python - # trino -pyyaml==6.0.1 - # via - # dask - # feast (setup.py) - # jupyter-events - # kubernetes - # pre-commit - # responses - # uvicorn -pyzmq==25.1.2 - # via - # ipykernel - # jupyter-client - # jupyter-server -redis==4.6.0 - # via feast (setup.py) -referencing==0.33.0 - # via - # jsonschema - # jsonschema-specifications - # jupyter-events -regex==2023.12.25 - # via feast (setup.py) -requests==2.31.0 - # via - # azure-core - # cachecontrol - # docker - # feast (setup.py) - # google-api-core - # google-cloud-bigquery - # google-cloud-storage - # great-expectations - # jupyterlab-server - # kubernetes - # moto - # msal - # requests-oauthlib - # responses - # snowflake-connector-python - # sphinx - # trino -requests-oauthlib==1.3.1 - # via kubernetes -responses==0.25.0 - # via moto -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rockset==2.1.0 - # via feast (setup.py) -rpds-py==0.18.0 - # via - # jsonschema - # referencing -rsa==4.9 - # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -s3transfer==0.10.0 - # via boto3 -scipy==1.12.0 - # via great-expectations -send2trash==1.8.2 - # via jupyter-server -six==1.16.0 - # via - # asttokens - # azure-core - # bleach - # geomet - # happybase - # isodate - # kubernetes - # mock - # pandavro - # python-dateutil - # rfc3339-validator - # thriftpy2 -sniffio==1.3.0 - # via - # anyio - # httpx -snowballstemmer==2.2.0 - # via sphinx -snowflake-connector-python[pandas]==3.7.0 - # via feast (setup.py) -sortedcontainers==2.4.0 - # via snowflake-connector-python -soupsieve==2.5 - # via beautifulsoup4 -sphinx==6.2.1 - # via feast (setup.py) -sphinxcontrib-applehelp==1.0.8 - # via sphinx -sphinxcontrib-devhelp==1.0.6 - # via sphinx -sphinxcontrib-htmlhelp==2.0.5 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.7 - # via sphinx -sphinxcontrib-serializinghtml==1.1.10 - # via sphinx -sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy -stack-data==0.6.3 - # via ipython -starlette==0.36.3 - # via fastapi -tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) -terminado==0.18.0 - # via - # jupyter-server - # jupyter-server-terminals -testcontainers==3.7.1 - # via feast (setup.py) -thriftpy2==0.4.17 - # via happybase -tinycss2==1.2.1 - # via nbconvert -toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via - # black - # build - # coverage - # jupyterlab - # mypy - # pip-tools - # pyproject-hooks - # pytest -tomlkit==0.12.3 - # via snowflake-connector-python -toolz==0.12.1 - # via - # altair - # dask - # partd -tornado==6.4 - # via - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado -tqdm==4.66.2 - # via - # feast (setup.py) - # great-expectations -traitlets==5.14.1 - # via - # comm - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab - # matplotlib-inline - # nbclient - # nbconvert - # nbformat -trino==0.327.0 - # via feast (setup.py) -typeguard==4.1.5 - # via feast (setup.py) -types-protobuf==3.19.22 - # via - # feast (setup.py) - # mypy-protobuf -types-pymysql==1.1.0.1 - # via feast (setup.py) -types-pyopenssl==24.0.0.20240130 - # via types-redis -types-python-dateutil==2.8.19.20240106 - # via - # arrow - # feast (setup.py) -types-pytz==2024.1.0.20240203 - # via feast (setup.py) -types-pyyaml==6.0.12.12 - # via feast (setup.py) -types-redis==4.6.0.20240106 - # via feast (setup.py) -types-requests==2.30.0.0 - # via feast (setup.py) -types-setuptools==69.0.0.20240125 - # via feast (setup.py) -types-tabulate==0.9.0.20240106 - # via feast (setup.py) -types-urllib3==1.26.25.14 - # via types-requests -typing-extensions==4.9.0 - # via - # anyio - # async-lru - # azure-core - # azure-storage-blob - # fastapi - # great-expectations - # mypy - # pydantic - # pydantic-core - # snowflake-connector-python - # sqlalchemy2-stubs - # uvicorn -tzlocal==5.2 - # via - # great-expectations - # trino -uri-template==1.3.0 - # via jsonschema -uritemplate==4.1.1 - # via google-api-python-client -urllib3==1.26.18 - # via - # botocore - # docker - # feast (setup.py) - # great-expectations - # kubernetes - # minio - # requests - # responses - # rockset -uvicorn[standard]==0.27.1 - # via feast (setup.py) -uvloop==0.19.0 - # via uvicorn -virtualenv==20.23.0 - # via - # feast (setup.py) - # pre-commit -volatile==2.1.0 - # via bowler -watchfiles==0.21.0 - # via uvicorn -wcwidth==0.2.13 - # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.7.0 - # via - # jupyter-server - # kubernetes -websockets==12.0 - # via uvicorn -werkzeug==3.0.1 - # via moto -wheel==0.42.0 - # via pip-tools -widgetsnbextension==4.0.10 - # via ipywidgets -wrapt==1.16.0 - # via testcontainers -xmltodict==0.13.0 - # via moto -zipp==3.17.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt deleted file mode 100644 index 3943662d01..0000000000 --- a/sdk/python/requirements/py3.11-requirements.txt +++ /dev/null @@ -1,234 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt -# -annotated-types==0.6.0 - # via pydantic -anyio==4.2.0 - # via - # httpx - # starlette - # watchfiles -appdirs==1.4.4 - # via fissix -attrs==23.2.0 - # via - # bowler - # jsonschema - # referencing -bowler==0.9.0 - # via feast (setup.py) -certifi==2024.2.2 - # via - # httpcore - # httpx - # requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # bowler - # dask - # feast (setup.py) - # moreorless - # uvicorn -cloudpickle==3.0.0 - # via dask -colorama==0.4.6 - # via feast (setup.py) -dask==2024.2.0 - # via feast (setup.py) -dill==0.3.8 - # via feast (setup.py) -exceptiongroup==1.2.0 - # via anyio -fastapi==0.109.2 - # via feast (setup.py) -fastavro==1.9.4 - # via - # feast (setup.py) - # pandavro -fissix==21.11.13 - # via bowler -fsspec==2024.2.0 - # via dask -greenlet==3.0.3 - # via sqlalchemy -grpcio==1.60.1 - # via - # feast (setup.py) - # grpcio-health-checking - # grpcio-reflection - # grpcio-tools -grpcio-health-checking==1.60.1 - # via feast (setup.py) -grpcio-reflection==1.60.1 - # via feast (setup.py) -grpcio-tools==1.60.1 - # via feast (setup.py) -gunicorn==21.2.0 - # via feast (setup.py) -h11==0.14.0 - # via - # httpcore - # uvicorn -httpcore==1.0.3 - # via httpx -httptools==0.6.1 - # via uvicorn -httpx==0.26.0 - # via feast (setup.py) -idna==3.6 - # via - # anyio - # httpx - # requests -importlib-metadata==6.11.0 - # via - # dask - # feast (setup.py) -importlib-resources==6.1.1 - # via feast (setup.py) -jinja2==3.1.3 - # via feast (setup.py) -jsonschema==4.21.1 - # via feast (setup.py) -jsonschema-specifications==2023.12.1 - # via jsonschema -locket==1.0.0 - # via partd -markupsafe==2.1.5 - # via jinja2 -mmh3==4.1.0 - # via feast (setup.py) -moreorless==0.4.0 - # via bowler -mypy==1.8.0 - # via sqlalchemy -mypy-extensions==1.0.0 - # via mypy -mypy-protobuf==3.1.0 - # via feast (setup.py) -numpy==1.24.4 - # via - # feast (setup.py) - # pandas - # pandavro - # pyarrow -packaging==23.2 - # via - # dask - # gunicorn -pandas==1.5.3 - # via - # feast (setup.py) - # pandavro -pandavro==1.5.2 - # via feast (setup.py) -partd==1.4.1 - # via dask -proto-plus==1.23.0 - # via feast (setup.py) -protobuf==4.23.3 - # via - # feast (setup.py) - # grpcio-health-checking - # grpcio-reflection - # grpcio-tools - # mypy-protobuf - # proto-plus -pyarrow==15.0.0 - # via feast (setup.py) -pydantic==2.6.1 - # via - # fastapi - # feast (setup.py) -pydantic-core==2.16.2 - # via pydantic -pygments==2.17.2 - # via feast (setup.py) -python-dateutil==2.8.2 - # via pandas -python-dotenv==1.0.1 - # via uvicorn -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via - # dask - # feast (setup.py) - # uvicorn -referencing==0.33.0 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via feast (setup.py) -rpds-py==0.18.0 - # via - # jsonschema - # referencing -six==1.16.0 - # via - # pandavro - # python-dateutil -sniffio==1.3.0 - # via - # anyio - # httpx -sqlalchemy[mypy]==1.4.51 - # via - # feast (setup.py) - # sqlalchemy -sqlalchemy2-stubs==0.0.2a38 - # via sqlalchemy -starlette==0.36.3 - # via fastapi -tabulate==0.9.0 - # via feast (setup.py) -tenacity==8.2.3 - # via feast (setup.py) -toml==0.10.2 - # via feast (setup.py) -tomli==2.0.1 - # via mypy -toolz==0.12.1 - # via - # dask - # partd -tqdm==4.66.2 - # via feast (setup.py) -typeguard==4.1.5 - # via feast (setup.py) -types-protobuf==4.24.0.20240129 - # via mypy-protobuf -typing-extensions==4.9.0 - # via - # anyio - # fastapi - # mypy - # pydantic - # pydantic-core - # sqlalchemy2-stubs - # uvicorn -urllib3==2.2.0 - # via requests -uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn -uvloop==0.19.0 - # via uvicorn -volatile==2.1.0 - # via bowler -watchfiles==0.21.0 - # via uvicorn -websockets==12.0 - # via uvicorn -zipp==3.17.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index afa43ec2a2..b006c1c621 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,14 +4,13 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # - alabaster==0.7.13 # via sphinx altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -35,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==3.1.2 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -66,11 +67,11 @@ black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -129,7 +130,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -219,9 +220,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -232,10 +233,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -250,7 +247,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -270,13 +267,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -288,15 +285,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -310,7 +307,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -318,11 +315,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==4.1.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==2.29.1 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -344,6 +347,7 @@ importlib-metadata==6.11.0 # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -384,7 +388,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -429,7 +433,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -445,6 +449,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -458,6 +464,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -472,7 +480,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -480,6 +488,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==0.6.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -494,7 +504,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -508,7 +518,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -518,6 +528,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -536,6 +547,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -553,6 +565,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -561,6 +574,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -571,7 +586,7 @@ pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) pkgutil-resolve-name==1.3.10 # via jsonschema @@ -601,7 +616,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -617,6 +632,7 @@ protobuf==4.23.3 # grpcio-status # grpcio-testing # grpcio-tools + # ibis-substrait # mypy-protobuf # proto-plus psutil==5.9.0 @@ -655,12 +671,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -669,6 +685,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -680,14 +697,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -720,6 +739,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -734,6 +754,7 @@ pytz==2024.1 # via # babel # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -741,6 +762,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -791,7 +813,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -819,6 +843,7 @@ six==1.16.0 # isodate # kubernetes # mock + # multipledispatch # pandavro # python-dateutil # rfc3339-validator @@ -829,7 +854,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -850,9 +875,13 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==10.6.4 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 @@ -889,6 +918,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -917,7 +947,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -937,11 +967,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -949,6 +979,7 @@ types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.9.0 # via + # annotated-types # anyio # async-lru # azure-core @@ -956,13 +987,16 @@ typing-extensions==4.9.0 # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic # pydantic-core + # rich # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 079064a9ec..541beecf0d 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -89,6 +89,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via # feast (setup.py) @@ -136,7 +137,7 @@ pkgutil-resolve-name==1.3.10 # via jsonschema proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -146,11 +147,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -184,7 +185,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -209,6 +212,7 @@ types-protobuf==4.24.0.20240129 # via mypy-protobuf typing-extensions==4.9.0 # via + # annotated-types # anyio # fastapi # mypy @@ -216,13 +220,12 @@ typing-extensions==4.9.0 # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0 diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 6c26f889e2..e17d545d38 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -10,7 +10,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -34,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==4.0 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -55,15 +57,17 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -122,7 +126,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -137,7 +141,7 @@ cryptography==41.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -213,9 +217,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -226,10 +230,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -244,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -264,13 +264,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -282,15 +282,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -304,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -312,11 +312,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -338,6 +344,7 @@ importlib-metadata==6.11.0 # jupyterlab-server # nbconvert # sphinx + # typeguard importlib-resources==6.1.1 # via feast (setup.py) iniconfig==2.0.0 @@ -374,7 +381,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -419,7 +426,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -435,6 +442,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -448,6 +457,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -462,7 +473,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -470,6 +481,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -484,7 +497,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -498,7 +511,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -508,6 +521,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -526,6 +540,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -543,6 +558,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -551,6 +567,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -559,7 +577,7 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -587,7 +605,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -605,6 +623,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -628,7 +647,10 @@ pyarrow==15.0.0 # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework pyasn1==0.5.1 # via # pyasn1-modules @@ -641,12 +663,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -655,6 +677,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -666,14 +689,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -706,6 +731,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -719,6 +745,7 @@ python-json-logger==2.0.7 pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -726,6 +753,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -776,7 +804,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -814,7 +844,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -835,13 +865,19 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -874,6 +910,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -902,7 +939,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -922,11 +959,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -941,6 +978,7 @@ typing-extensions==4.9.0 # black # fastapi # great-expectations + # ibis-framework # ipython # mypy # pydantic @@ -948,6 +986,7 @@ typing-extensions==4.9.0 # snowflake-connector-python # sqlalchemy2-stubs # starlette + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 182cb7ad07..12c14a03ae 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -38,7 +38,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -89,6 +89,7 @@ importlib-metadata==6.11.0 # via # dask # feast (setup.py) + # typeguard importlib-resources==6.1.1 # via feast (setup.py) jinja2==3.1.3 @@ -131,7 +132,7 @@ partd==1.4.1 # via dask proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -141,11 +142,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -179,7 +180,9 @@ sniffio==1.3.0 # anyio # httpx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy starlette==0.36.3 @@ -211,8 +214,9 @@ typing-extensions==4.9.0 # pydantic-core # sqlalchemy2-stubs # starlette + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 # via feast (setup.py) diff --git a/sdk/python/tests/unit/test_on_demand_substrait_transformation.py b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py new file mode 100644 index 0000000000..c9d30c5b7a --- /dev/null +++ b/sdk/python/tests/unit/test_on_demand_substrait_transformation.py @@ -0,0 +1,112 @@ +import os +import tempfile +from datetime import datetime, timedelta + +import pandas as pd + +from feast import Entity, FeatureStore, FeatureView, FileSource, RepoConfig +from feast.driver_test_data import create_driver_hourly_stats_df +from feast.field import Field +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + + +def test_ibis_pandas_parity(): + with tempfile.TemporaryDirectory() as data_dir: + store = FeatureStore( + config=RepoConfig( + project="test_on_demand_substrait_transformation", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=2, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db") + ), + ) + ) + + # Generate test data. + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + driver_stats_path = os.path.join(data_dir, "driver_stats.parquet") + driver_df.to_parquet(path=driver_stats_path, allow_truncated_timestamps=True) + + driver = Entity(name="driver", join_keys=["driver_id"]) + + driver_stats_source = FileSource( + name="driver_hourly_stats_source", + path=driver_stats_path, + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + @on_demand_feature_view( + sources=[driver_stats_fv], + schema=[Field(name="conv_rate_plus_acc", dtype=Float64)], + ) + def pandas_view(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_acc"] = inputs["conv_rate"] + inputs["acc_rate"] + return df + + from ibis.expr.types import Table + + @on_demand_feature_view( + sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], + schema=[Field(name="conv_rate_plus_acc_substrait", dtype=Float64)], + ) + def substrait_view(inputs: Table) -> Table: + return inputs.select( + (inputs["conv_rate"] + inputs["acc_rate"]).name( + "conv_rate_plus_acc_substrait" + ) + ) + + store.apply( + [driver, driver_stats_source, driver_stats_fv, substrait_view, pandas_view] + ) + + entity_df = pd.DataFrame.from_dict( + { + # entity's join key -> entity values + "driver_id": [1001, 1002, 1003], + # "event_timestamp" (reserved key) -> timestamps + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + } + ) + + training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "substrait_view:conv_rate_plus_acc_substrait", + "pandas_view:conv_rate_plus_acc", + ], + ).to_df() + + assert training_df["conv_rate_plus_acc"].equals( + training_df["conv_rate_plus_acc_substrait"] + ) diff --git a/setup.py b/setup.py index 818b9d66b1..3334d2e939 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<=2024.1.0"] -BYTEWAX_REQUIRED = ["bytewax==0.18.2", "docker>=5.0.2", "kubernetes<=20.13.0"] +BYTEWAX_REQUIRED = ["bytewax==0.15.1", "docker>=5.0.2", "kubernetes<=20.13.0"] SNOWFLAKE_REQUIRED = [ "snowflake-connector-python[pandas]>=3,<4", @@ -144,6 +144,11 @@ "hazelcast-python-client>=5.1", ] +IBIS_REQUIRED = [ + "ibis-framework", + "ibis-substrait" +] + CI_REQUIRED = ( [ "build", @@ -201,6 +206,7 @@ + AZURE_REQUIRED + ROCKSET_REQUIRED + HAZELCAST_REQUIRED + + IBIS_REQUIRED ) @@ -368,6 +374,7 @@ def run(self): "cassandra": CASSANDRA_REQUIRED, "hazelcast": HAZELCAST_REQUIRED, "rockset": ROCKSET_REQUIRED, + "ibis": IBIS_REQUIRED }, include_package_data=True, license="Apache", From daff5d883f0061dc9979b9424a1aec5e9079f9a6 Mon Sep 17 00:00:00 2001 From: tokoko Date: Sat, 24 Feb 2024 17:37:41 +0000 Subject: [PATCH 6/6] fix py310 locked requirements Signed-off-by: tokoko --- .../requirements/py3.10-ci-requirements.txt | 118 ++++++++++++------ .../requirements/py3.10-requirements.txt | 29 +++-- 2 files changed, 92 insertions(+), 55 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index f20bc05df9..45c4eb765d 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -10,7 +10,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # jupyter-server @@ -34,6 +34,8 @@ async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 # via redis +atpublic==4.0 + # via ibis-framework attrs==23.2.0 # via # bowler @@ -55,15 +57,17 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert +bidict==0.23.1 + # via ibis-framework black==22.12.0 # via feast (setup.py) bleach==6.1.0 # via nbconvert -boto3==1.34.42 +boto3==1.34.49 # via # feast (setup.py) # moto -botocore==1.34.42 +botocore==1.34.49 # via # boto3 # moto @@ -122,7 +126,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov cryptography==41.0.7 # via @@ -137,7 +141,7 @@ cryptography==41.0.7 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) db-dtypes==1.2.0 # via google-cloud-bigquery @@ -213,9 +217,9 @@ google-api-core[grpc]==2.17.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.118.0 +google-api-python-client==2.119.0 # via firebase-admin -google-auth==2.27.0 +google-auth==2.28.1 # via # google-api-core # google-api-python-client @@ -226,10 +230,6 @@ google-auth==2.27.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-cloud-bigquery[pandas]==3.12.0 - # via - # feast (setup.py) - # google-cloud-bigquery -google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) google-cloud-bigquery-storage==2.24.0 # via feast (setup.py) @@ -244,7 +244,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-datastore==2.19.0 # via feast (setup.py) -google-cloud-firestore==2.14.0 +google-cloud-firestore==2.15.0 # via firebase-admin google-cloud-storage==2.14.0 # via @@ -264,13 +264,13 @@ googleapis-common-protos[grpc]==1.62.0 # google-api-core # grpc-google-iam-v1 # grpcio-status -great-expectations==0.18.8 +great-expectations==0.18.9 # via feast (setup.py) greenlet==3.0.3 # via sqlalchemy grpc-google-iam-v1==0.13.0 # via google-cloud-bigtable -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # google-api-core @@ -282,15 +282,15 @@ grpcio==1.60.1 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-status==1.60.1 +grpcio-status==1.62.0 # via google-api-core -grpcio-testing==1.60.1 +grpcio-testing==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -304,7 +304,7 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.3.2 # via feast (setup.py) -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httplib2==0.22.0 # via @@ -312,11 +312,17 @@ httplib2==0.22.0 # google-auth-httplib2 httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via # feast (setup.py) # jupyterlab -identify==2.5.34 +ibis-framework==8.0.0 + # via + # feast (setup.py) + # ibis-substrait +ibis-substrait==3.2.0 + # via feast (setup.py) +identify==2.5.35 # via pre-commit idna==3.6 # via @@ -337,7 +343,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.2 # via jupyterlab -ipython==8.21.0 +ipython==8.22.1 # via # great-expectations # ipykernel @@ -367,7 +373,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -412,7 +418,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -428,6 +434,8 @@ locket==1.0.0 # via partd makefun==1.15.2 # via great-expectations +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # jinja2 @@ -441,6 +449,8 @@ matplotlib-inline==0.1.6 # ipython mccabe==0.7.0 # via flake8 +mdurl==0.1.2 + # via markdown-it-py minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -455,7 +465,7 @@ moreorless==0.4.0 # via bowler moto==4.2.14 # via feast (setup.py) -msal==1.26.0 +msal==1.27.0 # via # azure-identity # msal-extensions @@ -463,6 +473,8 @@ msal-extensions==1.1.0 # via azure-identity msgpack==1.0.7 # via cachecontrol +multipledispatch==1.0.0 + # via ibis-framework multiprocess==0.70.16 # via bytewax mypy==1.8.0 @@ -477,7 +489,7 @@ mypy-protobuf==3.1.0 # via feast (setup.py) nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via jupyter-server nbformat==5.9.2 # via @@ -491,7 +503,7 @@ nodeenv==1.8.0 # via pre-commit notebook==7.1.0 # via great-expectations -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via # jupyterlab # notebook @@ -501,6 +513,7 @@ numpy==1.24.4 # db-dtypes # feast (setup.py) # great-expectations + # ibis-framework # pandas # pandavro # pyarrow @@ -519,6 +532,7 @@ packaging==23.2 # google-cloud-bigquery # great-expectations # gunicorn + # ibis-substrait # ipykernel # jupyter-server # jupyterlab @@ -536,6 +550,7 @@ pandas==1.5.3 # feast (setup.py) # google-cloud-bigquery # great-expectations + # ibis-framework # pandavro # snowflake-connector-python pandavro==1.5.2 @@ -544,6 +559,8 @@ pandocfilters==1.5.1 # via nbconvert parso==0.8.3 # via jedi +parsy==2.1 + # via ibis-framework partd==1.4.1 # via dask pathspec==0.12.1 @@ -552,7 +569,7 @@ pbr==6.0.0 # via mock pexpect==4.9.0 # via ipython -pip-tools==7.3.0 +pip-tools==7.4.0 # via feast (setup.py) platformdirs==3.11.0 # via @@ -580,7 +597,7 @@ proto-plus==1.23.0 # google-cloud-bigtable # google-cloud-datastore # google-cloud-firestore -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # google-api-core @@ -598,6 +615,7 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus + # substrait psutil==5.9.0 # via # feast (setup.py) @@ -621,7 +639,10 @@ pyarrow==15.0.0 # db-dtypes # feast (setup.py) # google-cloud-bigquery + # ibis-framework # snowflake-connector-python +pyarrow-hotfix==0.6 + # via ibis-framework pyasn1==0.5.1 # via # pyasn1-modules @@ -634,12 +655,12 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) # great-expectations -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pyflakes==3.0.1 # via flake8 @@ -648,6 +669,7 @@ pygments==2.17.2 # feast (setup.py) # ipython # nbconvert + # rich # sphinx pyjwt[crypto]==2.8.0 # via @@ -659,14 +681,16 @@ pymysql==1.1.0 # via feast (setup.py) pyodbc==5.1.0 # via feast (setup.py) -pyopenssl==23.3.0 +pyopenssl==24.0.0 # via snowflake-connector-python pyparsing==3.1.1 # via # great-expectations # httplib2 pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools pyspark==3.5.0 # via feast (setup.py) pytest==7.4.4 @@ -699,6 +723,7 @@ python-dateutil==2.8.2 # botocore # google-cloud-bigquery # great-expectations + # ibis-framework # jupyter-client # kubernetes # moto @@ -712,6 +737,7 @@ python-json-logger==2.0.7 pytz==2024.1 # via # great-expectations + # ibis-framework # pandas # snowflake-connector-python # trino @@ -719,6 +745,7 @@ pyyaml==6.0.1 # via # dask # feast (setup.py) + # ibis-substrait # jupyter-events # kubernetes # pre-commit @@ -769,7 +796,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rockset==2.1.0 +rich==13.7.0 + # via ibis-framework +rockset==2.1.1 # via feast (setup.py) rpds-py==0.18.0 # via @@ -805,7 +834,7 @@ sniffio==1.3.0 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.7.0 +snowflake-connector-python[pandas]==3.7.1 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -826,13 +855,19 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy[mypy]==1.4.51 - # via feast (setup.py) + # via + # feast (setup.py) + # sqlalchemy sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy +sqlglot==20.11.0 + # via ibis-framework stack-data==0.6.3 # via ipython starlette==0.36.3 # via fastapi +substrait==0.12.1 + # via ibis-substrait tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 @@ -865,6 +900,7 @@ toolz==0.12.1 # via # altair # dask + # ibis-framework # partd tornado==6.4 # via @@ -893,7 +929,7 @@ traitlets==5.14.1 # nbclient # nbconvert # nbformat -trino==0.327.0 +trino==0.328.0 # via feast (setup.py) typeguard==4.1.5 # via feast (setup.py) @@ -913,11 +949,11 @@ types-pytz==2024.1.0.20240203 # via feast (setup.py) types-pyyaml==6.0.12.12 # via feast (setup.py) -types-redis==4.6.0.20240106 +types-redis==4.6.0.20240218 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==69.0.0.20240125 +types-setuptools==69.1.0.20240223 # via feast (setup.py) types-tabulate==0.9.0.20240106 # via feast (setup.py) @@ -931,11 +967,13 @@ typing-extensions==4.9.0 # azure-storage-blob # fastapi # great-expectations + # ibis-framework # mypy # pydantic # pydantic-core # snowflake-connector-python # sqlalchemy2-stubs + # typeguard # uvicorn tzlocal==5.2 # via diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 3943662d01..7141cd0f25 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -anyio==4.2.0 +anyio==4.3.0 # via # httpx # starlette @@ -38,7 +38,7 @@ cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2024.2.0 +dask==2024.2.1 # via feast (setup.py) dill==0.3.8 # via feast (setup.py) @@ -56,17 +56,17 @@ fsspec==2024.2.0 # via dask greenlet==3.0.3 # via sqlalchemy -grpcio==1.60.1 +grpcio==1.62.0 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.60.1 +grpcio-health-checking==1.62.0 # via feast (setup.py) -grpcio-reflection==1.60.1 +grpcio-reflection==1.62.0 # via feast (setup.py) -grpcio-tools==1.60.1 +grpcio-tools==1.62.0 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -74,11 +74,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.3 +httpcore==1.0.4 # via httpx httptools==0.6.1 # via uvicorn -httpx==0.26.0 +httpx==0.27.0 # via feast (setup.py) idna==3.6 # via @@ -131,7 +131,7 @@ partd==1.4.1 # via dask proto-plus==1.23.0 # via feast (setup.py) -protobuf==4.23.3 +protobuf==4.23.4 # via # feast (setup.py) # grpcio-health-checking @@ -141,11 +141,11 @@ protobuf==4.23.3 # proto-plus pyarrow==15.0.0 # via feast (setup.py) -pydantic==2.6.1 +pydantic==2.6.2 # via # fastapi # feast (setup.py) -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pygments==2.17.2 # via feast (setup.py) @@ -212,13 +212,12 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # sqlalchemy2-stubs + # typeguard # uvicorn -urllib3==2.2.0 +urllib3==2.2.1 # via requests uvicorn[standard]==0.27.1 - # via - # feast (setup.py) - # uvicorn + # via feast (setup.py) uvloop==0.19.0 # via uvicorn volatile==2.1.0