From d3868c5ceec62bab9e12e115c5167c88df0a2745 Mon Sep 17 00:00:00 2001 From: Abhin Chhabra Date: Mon, 18 Jul 2022 21:22:51 -0400 Subject: [PATCH] chore: Upgrade GCP dependencies (#2945) * Upgrade GCP dependencies. Signed-off-by: Abhin Chhabra * `entity_df` in `get_historical_features` should be tzaware Signed-off-by: Abhin Chhabra * Linting fixes Signed-off-by: Abhin Chhabra * Update 3.9 reqs Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang * Remove unnecessary code coverage Signed-off-by: Kevin Zhang * Fix Signed-off-by: Kevin Zhang Co-authored-by: Kevin Zhang --- sdk/python/feast/feature_store.py | 10 +- .../feast/infra/offline_stores/bigquery.py | 5 +- sdk/python/feast/utils.py | 9 ++ .../requirements/py3.10-ci-requirements.txt | 102 ++++++++++-------- .../requirements/py3.10-requirements.txt | 18 ++-- .../requirements/py3.8-ci-requirements.txt | 76 +++++++------ .../requirements/py3.8-requirements.txt | 10 +- .../requirements/py3.9-ci-requirements.txt | 102 ++++++++++-------- .../requirements/py3.9-requirements.txt | 18 ++-- .../universal/data_sources/bigquery.py | 5 + setup.py | 7 +- 11 files changed, 203 insertions(+), 159 deletions(-) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 1eeb8c22d3..4856a46f8c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1057,17 +1057,17 @@ def get_historical_features( # Check that the right request data is present in the entity_df if type(entity_df) == pd.DataFrame: - entity_pd_df = cast(pd.DataFrame, entity_df) + entity_df = utils.make_df_tzaware(cast(pd.DataFrame, entity_df)) for fv in request_feature_views: for feature in fv.features: - if feature.name not in entity_pd_df.columns: + if feature.name not in entity_df.columns: raise RequestDataNotFoundInEntityDfException( feature_name=feature.name, feature_view_name=fv.name ) for odfv in on_demand_feature_views: odfv_request_data_schema = odfv.get_request_data_schema() for feature_name in odfv_request_data_schema.keys(): - if feature_name not in entity_pd_df.columns: + if feature_name not in entity_df.columns: raise RequestDataNotFoundInEntityDfException( feature_name=feature_name, feature_view_name=odfv.name, ) @@ -2273,7 +2273,7 @@ def _teardown_go_server(self): @log_exceptions_and_usage def write_logged_features( - self, logs: Union[pa.Table, Path], source: Union[FeatureService] + self, logs: Union[pa.Table, Path], source: FeatureService ): """ Write logs produced by a source (currently only feature service is supported as a source) @@ -2302,7 +2302,7 @@ def write_logged_features( @log_exceptions_and_usage def validate_logged_features( self, - source: Union[FeatureService], + source: FeatureService, start: datetime, end: datetime, reference: ValidationReference, diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 3bf340acf9..982045607d 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -424,7 +424,7 @@ def to_bigquery( job_config: bigquery.QueryJobConfig = None, timeout: int = 1800, retry_cadence: int = 10, - ) -> Optional[str]: + ) -> str: """ Triggers the execution of a historical feature retrieval query and exports the results to a BigQuery table. Runs for a maximum amount of time specified by the timeout parameter (defaulting to 30 minutes). @@ -567,7 +567,7 @@ def _wait_until_done(bq_job): finally: if client.get_job(bq_job).state in ["PENDING", "RUNNING"]: - client.cancel_job(bq_job) + client.cancel_job(bq_job.job_id) raise BigQueryJobCancelled(job_id=bq_job.job_id) if bq_job.exception(): @@ -601,6 +601,7 @@ def _upload_entity_df( client: Client, table_name: str, entity_df: Union[pd.DataFrame, str], ) -> Table: """Uploads a Pandas entity dataframe into a BigQuery table and returns the resulting table""" + job: Union[bigquery.job.query.QueryJob, bigquery.job.load.LoadJob] if isinstance(entity_df, str): job = client.query(f"CREATE TABLE {table_name} AS ({entity_df})") diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 9f18da38cd..af22fbca3f 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -28,6 +28,15 @@ def make_tzaware(t: datetime) -> datetime: return t +def make_df_tzaware(t: pd.DataFrame) -> pd.DataFrame: + """Make all datetime type columns tzaware; leave everything else intact.""" + df = t.copy() # don't modify incoming dataframe inplace + for column in df.columns: + if pd.api.types.is_datetime64_any_dtype(df[column]): + df[column] = pd.to_datetime(df[column], utc=True) + return df + + def to_naive_utc(ts: datetime) -> datetime: if ts.tzinfo is None: return ts diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index e0b05ecaa0..9e47356000 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # -absl-py==1.1.0 +absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 # via @@ -59,7 +59,7 @@ attrs==21.4.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.24.1 +azure-core==1.24.2 # via # adlfs # azure-identity @@ -69,7 +69,7 @@ azure-datalake-store==0.0.52 # via adlfs azure-identity==1.10.0 # via adlfs -azure-storage-blob==12.12.0 +azure-storage-blob==12.13.0 # via adlfs babel==2.10.3 # via sphinx @@ -95,7 +95,7 @@ build==0.8.0 # pip-tools cachecontrol==0.12.11 # via firebase-admin -cachetools==4.2.4 +cachetools==5.2.0 # via google-auth certifi==2022.6.15 # via @@ -103,7 +103,7 @@ certifi==2022.6.15 # msrest # requests # snowflake-connector-python -cffi==1.15.0 +cffi==1.15.1 # via # azure-datalake-store # cryptography @@ -130,7 +130,7 @@ colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.1 +coverage[toml]==6.4.2 # via pytest-cov cryptography==35.0.0 # via @@ -147,6 +147,8 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations +db-dtypes==1.0.2 + # via google-cloud-bigquery decorator==5.1.1 # via # gcsfs @@ -157,7 +159,7 @@ deprecation==2.1.0 # via testcontainers dill==0.3.5.1 # via feast (setup.py) -distlib==0.3.4 +distlib==0.3.5 # via virtualenv docker==5.0.3 # via @@ -173,17 +175,17 @@ execnet==1.9.0 # via pytest-xdist executing==0.8.3 # via stack-data -fastapi==0.78.0 +fastapi==0.79.0 # via feast (setup.py) fastavro==1.5.2 # via # feast (setup.py) # pandavro -fastjsonschema==2.15.3 +fastjsonschema==2.16.1 # via nbformat filelock==3.7.1 # via virtualenv -firebase-admin==4.5.2 +firebase-admin==5.2.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -201,7 +203,7 @@ fsspec==2022.1.0 # s3fs gcsfs==2022.1.0 # via feast (setup.py) -google-api-core[grpc]==1.31.6 +google-api-core[grpc]==2.8.2 # via # feast (setup.py) # firebase-admin @@ -211,9 +213,10 @@ google-api-core[grpc]==1.31.6 # google-cloud-core # google-cloud-datastore # google-cloud-firestore -google-api-python-client==2.52.0 + # google-cloud-storage +google-api-python-client==2.53.0 # via firebase-admin -google-auth==1.35.0 +google-auth==2.9.1 # via # gcsfs # google-api-core @@ -226,36 +229,38 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==0.5.2 # via gcsfs -google-cloud-bigquery==2.34.4 - # via feast (setup.py) -google-cloud-bigquery-storage==2.13.2 +google-cloud-bigquery[pandas]==3.2.0 # via feast (setup.py) -google-cloud-core==1.7.2 +google-cloud-bigquery-storage==2.14.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-core==2.3.2 + # via + # google-cloud-bigquery # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.7.1 +google-cloud-datastore==2.8.0 # via feast (setup.py) -google-cloud-firestore==2.5.3 +google-cloud-firestore==2.6.0 # via firebase-admin -google-cloud-storage==1.40.0 +google-cloud-storage==2.4.0 # via # feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==1.3.3 +google-resumable-media==2.3.3 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core + # grpcio-status # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) @@ -265,10 +270,13 @@ grpcio==1.47.0 # google-api-core # google-cloud-bigquery # grpcio-reflection + # grpcio-status # grpcio-testing # grpcio-tools grpcio-reflection==1.47.0 # via feast (setup.py) +grpcio-status==1.47.0 + # via google-api-core grpcio-testing==1.47.0 # via feast (setup.py) grpcio-tools==1.47.0 @@ -293,7 +301,7 @@ idna==3.3 # requests # snowflake-connector-python # yarl -imagesize==1.3.0 +imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 # via great-expectations @@ -322,13 +330,13 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.1 +jsonschema==4.7.2 # via # altair # feast (setup.py) # great-expectations # nbformat -jupyter-core==4.10.0 +jupyter-core==4.11.1 # via nbformat locket==1.0.0 # via partd @@ -342,7 +350,7 @@ mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.3 +mistune==2.0.4 # via great-expectations mmh3==3.0.0 # via feast (setup.py) @@ -384,9 +392,10 @@ nbformat==5.4.0 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.22.0 +numpy==1.23.1 # via # altair + # db-dtypes # feast (setup.py) # great-expectations # pandas @@ -401,8 +410,8 @@ packaging==21.3 # via # build # dask + # db-dtypes # deprecation - # google-api-core # google-cloud-bigquery # great-expectations # pytest @@ -411,7 +420,9 @@ packaging==21.3 pandas==1.4.3 # via # altair + # db-dtypes # feast (setup.py) + # google-cloud-bigquery # great-expectations # pandavro # snowflake-connector-python @@ -431,7 +442,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.7.0 +pip-tools==6.8.0 # via feast (setup.py) platformdirs==2.5.2 # via virtualenv @@ -439,9 +450,9 @@ pluggy==1.0.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.4.0 +portalocker==2.5.1 # via msal-extensions -pre-commit==2.19.0 +pre-commit==2.20.0 # via feast (setup.py) prompt-toolkit==3.0.30 # via ipython @@ -462,6 +473,7 @@ protobuf==3.20.1 # google-cloud-firestore # googleapis-common-protos # grpcio-reflection + # grpcio-status # grpcio-testing # grpcio-tools # mypy-protobuf @@ -485,7 +497,9 @@ py4j==0.10.9.5 # via pyspark pyarrow==6.0.1 # via + # db-dtypes # feast (setup.py) + # google-cloud-bigquery # snowflake-connector-python pyasn1==0.4.8 # via @@ -568,7 +582,6 @@ python-dotenv==0.20.0 pytz==2022.1 # via # babel - # google-api-core # great-expectations # moto # pandas @@ -584,7 +597,7 @@ pyyaml==6.0 # uvicorn redis==4.2.2 # via feast (setup.py) -regex==2022.6.2 +regex==2022.7.9 # via black requests==2.28.1 # via @@ -627,11 +640,8 @@ six==1.16.0 # via # azure-core # azure-identity - # google-api-core # google-auth # google-auth-httplib2 - # google-cloud-core - # google-resumable-media # grpcio # happybase # mock @@ -695,7 +705,7 @@ tomli==2.0.1 # mypy # pep517 # pytest -toolz==0.11.2 +toolz==0.12.0 # via # altair # dask @@ -710,7 +720,7 @@ traitlets==5.3.0 # jupyter-core # matplotlib-inline # nbformat -trino==0.313.0 +trino==0.314.0 # via feast (setup.py) typed-ast==1.5.4 # via black @@ -724,19 +734,19 @@ types-python-dateutil==2.8.18 # via feast (setup.py) types-pytz==2022.1.1 # via feast (setup.py) -types-pyyaml==6.0.9 +types-pyyaml==6.0.10 # via feast (setup.py) -types-redis==4.3.3 +types-redis==4.3.4 # via feast (setup.py) -types-requests==2.28.0 +types-requests==2.28.2 # via feast (setup.py) -types-setuptools==57.4.18 +types-setuptools==63.2.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.15 +types-urllib3==1.26.16 # via types-requests -typing-extensions==4.2.0 +typing-extensions==4.3.0 # via # azure-core # great-expectations @@ -749,7 +759,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.9 +urllib3==1.26.10 # via # botocore # feast (setup.py) @@ -786,7 +796,7 @@ xmltodict==0.13.0 # via moto yarl==1.7.2 # via aiohttp -zipp==3.8.0 +zipp==3.8.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index d4a4425aec..0440a171ed 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=sdk/python/requirements/py3.10-requirements.txt # -absl-py==1.1.0 +absl-py==1.2.0 # via tensorflow-metadata anyio==3.6.1 # via @@ -38,7 +38,7 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.78.0 +fastapi==0.79.0 # via feast (setup.py) fastavro==1.5.2 # via @@ -50,9 +50,9 @@ fsspec==2022.5.0 # via dask google-api-core==2.8.2 # via feast (setup.py) -google-auth==2.9.0 +google-auth==2.9.1 # via google-api-core -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core @@ -73,7 +73,7 @@ idna==3.3 # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.6.1 +jsonschema==4.7.2 # via feast (setup.py) locket==1.0.0 # via partd @@ -87,7 +87,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.22.0 +numpy==1.23.1 # via # feast (setup.py) # pandas @@ -170,7 +170,7 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.11.2 +toolz==0.12.0 # via # dask # partd @@ -178,12 +178,12 @@ tqdm==4.64.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -typing-extensions==4.2.0 +typing-extensions==4.3.0 # via # mypy # pydantic # sqlalchemy2-stubs -urllib3==1.26.9 +urllib3==1.26.10 # via requests uvicorn[standard]==0.18.2 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 813420a121..67db3a978f 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -59,7 +59,7 @@ attrs==21.4.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.24.1 +azure-core==1.24.2 # via # adlfs # azure-identity @@ -69,7 +69,7 @@ azure-datalake-store==0.0.52 # via adlfs azure-identity==1.10.0 # via adlfs -azure-storage-blob==12.12.0 +azure-storage-blob==12.13.0 # via adlfs babel==2.10.3 # via sphinx @@ -99,7 +99,7 @@ build==0.8.0 # pip-tools cachecontrol==0.12.11 # via firebase-admin -cachetools==4.2.4 +cachetools==5.2.0 # via google-auth certifi==2022.6.15 # via @@ -107,7 +107,7 @@ certifi==2022.6.15 # msrest # requests # snowflake-connector-python -cffi==1.15.0 +cffi==1.15.1 # via # azure-datalake-store # cryptography @@ -151,6 +151,8 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations +db-dtypes==1.0.2 + # via google-cloud-bigquery decorator==5.1.1 # via # gcsfs @@ -187,7 +189,7 @@ fastjsonschema==2.15.3 # via nbformat filelock==3.7.1 # via virtualenv -firebase-admin==4.5.2 +firebase-admin==5.2.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -205,7 +207,7 @@ fsspec==2022.1.0 # s3fs gcsfs==2022.1.0 # via feast (setup.py) -google-api-core[grpc]==1.31.6 +google-api-core[grpc]==2.8.2 # via # feast (setup.py) # firebase-admin @@ -215,9 +217,10 @@ google-api-core[grpc]==1.31.6 # google-cloud-core # google-cloud-datastore # google-cloud-firestore + # google-cloud-storage google-api-python-client==2.52.0 # via firebase-admin -google-auth==1.35.0 +google-auth==2.9.0 # via # gcsfs # google-api-core @@ -230,29 +233,30 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==0.5.2 # via gcsfs -google-cloud-bigquery==2.34.4 - # via feast (setup.py) -google-cloud-bigquery-storage==2.13.2 +google-cloud-bigquery[pandas]==3.2.0 # via feast (setup.py) -google-cloud-core==1.7.2 +google-cloud-bigquery-storage==2.14.0 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-core==2.3.1 + # via + # google-cloud-bigquery # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.7.1 +google-cloud-datastore==2.7.2 # via feast (setup.py) google-cloud-firestore==2.5.3 # via firebase-admin -google-cloud-storage==1.40.0 +google-cloud-storage==2.4.0 # via # feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==1.3.3 +google-resumable-media==2.3.3 # via # google-cloud-bigquery # google-cloud-storage @@ -260,6 +264,7 @@ googleapis-common-protos==1.56.3 # via # feast (setup.py) # google-api-core + # grpcio-status # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) @@ -269,10 +274,13 @@ grpcio==1.47.0 # google-api-core # google-cloud-bigquery # grpcio-reflection + # grpcio-status # grpcio-testing # grpcio-tools grpcio-reflection==1.47.0 # via feast (setup.py) +grpcio-status==1.47.0 + # via google-api-core grpcio-testing==1.47.0 # via feast (setup.py) grpcio-tools==1.47.0 @@ -297,7 +305,7 @@ idna==3.3 # requests # snowflake-connector-python # yarl -imagesize==1.3.0 +imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 # via great-expectations @@ -328,13 +336,13 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.1 +jsonschema==4.7.1 # via # altair # feast (setup.py) # great-expectations # nbformat -jupyter-core==4.10.0 +jupyter-core==4.11.1 # via nbformat locket==1.0.0 # via partd @@ -390,9 +398,10 @@ nbformat==5.4.0 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.23.0 +numpy==1.23.1 # via # altair + # db-dtypes # feast (setup.py) # great-expectations # pandas @@ -407,8 +416,8 @@ packaging==21.3 # via # build # dask + # db-dtypes # deprecation - # google-api-core # google-cloud-bigquery # great-expectations # pytest @@ -417,7 +426,9 @@ packaging==21.3 pandas==1.4.3 # via # altair + # db-dtypes # feast (setup.py) + # google-cloud-bigquery # great-expectations # pandavro # snowflake-connector-python @@ -437,7 +448,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.7.0 +pip-tools==6.8.0 # via feast (setup.py) platformdirs==2.5.2 # via virtualenv @@ -445,9 +456,9 @@ pluggy==1.0.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.4.0 +portalocker==2.5.1 # via msal-extensions -pre-commit==2.19.0 +pre-commit==2.20.0 # via feast (setup.py) prompt-toolkit==3.0.30 # via ipython @@ -468,6 +479,7 @@ protobuf==3.20.1 # google-cloud-firestore # googleapis-common-protos # grpcio-reflection + # grpcio-status # grpcio-testing # grpcio-tools # mypy-protobuf @@ -491,7 +503,9 @@ py4j==0.10.9.5 # via pyspark pyarrow==6.0.1 # via + # db-dtypes # feast (setup.py) + # google-cloud-bigquery # snowflake-connector-python pyasn1==0.4.8 # via @@ -574,7 +588,6 @@ python-dotenv==0.20.0 pytz==2022.1 # via # babel - # google-api-core # great-expectations # moto # pandas @@ -590,7 +603,7 @@ pyyaml==6.0 # uvicorn redis==4.2.2 # via feast (setup.py) -regex==2022.6.2 +regex==2022.7.9 # via black requests==2.28.1 # via @@ -635,11 +648,8 @@ six==1.16.0 # via # azure-core # azure-identity - # google-api-core # google-auth # google-auth-httplib2 - # google-cloud-core - # google-resumable-media # grpcio # happybase # mock @@ -703,7 +713,7 @@ tomli==2.0.1 # mypy # pep517 # pytest -toolz==0.11.2 +toolz==0.12.0 # via # altair # dask @@ -718,7 +728,7 @@ traitlets==5.3.0 # jupyter-core # matplotlib-inline # nbformat -trino==0.313.0 +trino==0.314.0 # via feast (setup.py) typed-ast==1.5.4 # via black @@ -738,13 +748,13 @@ types-redis==4.3.3 # via feast (setup.py) types-requests==2.28.0 # via feast (setup.py) -types-setuptools==57.4.18 +types-setuptools==62.6.1 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.15 +types-urllib3==1.26.16 # via types-requests -typing-extensions==4.2.0 +typing-extensions==4.3.0 # via # aioitertools # azure-core @@ -759,7 +769,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.9 +urllib3==1.26.10 # via # botocore # feast (setup.py) diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 56e5e3adba..a09289f662 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -75,7 +75,7 @@ importlib-resources==5.8.0 # via jsonschema jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.6.1 +jsonschema==4.7.1 # via feast (setup.py) locket==1.0.0 # via partd @@ -89,7 +89,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.23.0 +numpy==1.23.1 # via # feast (setup.py) # pandas @@ -172,7 +172,7 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.11.2 +toolz==0.12.0 # via # dask # partd @@ -180,13 +180,13 @@ tqdm==4.64.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -typing-extensions==4.2.0 +typing-extensions==4.3.0 # via # mypy # pydantic # sqlalchemy2-stubs # starlette -urllib3==1.26.9 +urllib3==1.26.10 # via requests uvicorn[standard]==0.18.2 # via feast (setup.py) diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index fe1b8b2e13..8be0c36052 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # -absl-py==1.1.0 +absl-py==1.2.0 # via tensorflow-metadata adal==1.2.7 # via @@ -59,7 +59,7 @@ attrs==21.4.0 # pytest avro==1.10.0 # via feast (setup.py) -azure-core==1.24.1 +azure-core==1.24.2 # via # adlfs # azure-identity @@ -69,7 +69,7 @@ azure-datalake-store==0.0.52 # via adlfs azure-identity==1.10.0 # via adlfs -azure-storage-blob==12.12.0 +azure-storage-blob==12.13.0 # via adlfs babel==2.10.3 # via sphinx @@ -95,7 +95,7 @@ build==0.8.0 # pip-tools cachecontrol==0.12.11 # via firebase-admin -cachetools==4.2.4 +cachetools==5.2.0 # via google-auth certifi==2022.6.15 # via @@ -103,7 +103,7 @@ certifi==2022.6.15 # msrest # requests # snowflake-connector-python -cffi==1.15.0 +cffi==1.15.1 # via # azure-datalake-store # cryptography @@ -130,7 +130,7 @@ colorama==0.4.5 # via # feast (setup.py) # great-expectations -coverage[toml]==6.4.1 +coverage[toml]==6.4.2 # via pytest-cov cryptography==35.0.0 # via @@ -147,6 +147,8 @@ dask==2022.1.1 # via feast (setup.py) dataclasses==0.6 # via great-expectations +db-dtypes==1.0.2 + # via google-cloud-bigquery decorator==5.1.1 # via # gcsfs @@ -157,7 +159,7 @@ deprecation==2.1.0 # via testcontainers dill==0.3.5.1 # via feast (setup.py) -distlib==0.3.4 +distlib==0.3.5 # via virtualenv docker==5.0.3 # via @@ -173,17 +175,17 @@ execnet==1.9.0 # via pytest-xdist executing==0.8.3 # via stack-data -fastapi==0.78.0 +fastapi==0.79.0 # via feast (setup.py) fastavro==1.5.2 # via # feast (setup.py) # pandavro -fastjsonschema==2.15.3 +fastjsonschema==2.16.1 # via nbformat filelock==3.7.1 # via virtualenv -firebase-admin==4.5.2 +firebase-admin==5.2.0 # via feast (setup.py) fissix==21.11.13 # via bowler @@ -201,7 +203,7 @@ fsspec==2022.1.0 # s3fs gcsfs==2022.1.0 # via feast (setup.py) -google-api-core[grpc]==1.31.6 +google-api-core[grpc]==2.8.2 # via # feast (setup.py) # firebase-admin @@ -211,9 +213,10 @@ google-api-core[grpc]==1.31.6 # google-cloud-core # google-cloud-datastore # google-cloud-firestore -google-api-python-client==2.52.0 + # google-cloud-storage +google-api-python-client==2.53.0 # via firebase-admin -google-auth==1.35.0 +google-auth==2.9.1 # via # gcsfs # google-api-core @@ -226,36 +229,38 @@ google-auth-httplib2==0.1.0 # via google-api-python-client google-auth-oauthlib==0.5.2 # via gcsfs -google-cloud-bigquery==2.34.4 - # via feast (setup.py) -google-cloud-bigquery-storage==2.13.2 +google-cloud-bigquery[pandas]==3.2.0 # via feast (setup.py) -google-cloud-core==1.7.2 +google-cloud-bigquery-storage==2.14.1 # via # feast (setup.py) # google-cloud-bigquery +google-cloud-core==2.3.2 + # via + # google-cloud-bigquery # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-cloud-datastore==2.7.1 +google-cloud-datastore==2.8.0 # via feast (setup.py) -google-cloud-firestore==2.5.3 +google-cloud-firestore==2.6.0 # via firebase-admin -google-cloud-storage==1.40.0 +google-cloud-storage==2.4.0 # via # feast (setup.py) # firebase-admin # gcsfs google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==1.3.3 +google-resumable-media==2.3.3 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core + # grpcio-status # tensorflow-metadata great-expectations==0.14.13 # via feast (setup.py) @@ -265,10 +270,13 @@ grpcio==1.47.0 # google-api-core # google-cloud-bigquery # grpcio-reflection + # grpcio-status # grpcio-testing # grpcio-tools grpcio-reflection==1.47.0 # via feast (setup.py) +grpcio-status==1.47.0 + # via google-api-core grpcio-testing==1.47.0 # via feast (setup.py) grpcio-tools==1.47.0 @@ -293,7 +301,7 @@ idna==3.3 # requests # snowflake-connector-python # yarl -imagesize==1.3.0 +imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 # via great-expectations @@ -322,13 +330,13 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.1 +jsonschema==4.7.2 # via # altair # feast (setup.py) # great-expectations # nbformat -jupyter-core==4.10.0 +jupyter-core==4.11.1 # via nbformat locket==1.0.0 # via partd @@ -342,7 +350,7 @@ mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==2.0.3 +mistune==2.0.4 # via great-expectations mmh3==3.0.0 # via feast (setup.py) @@ -384,9 +392,10 @@ nbformat==5.4.0 # via great-expectations nodeenv==1.7.0 # via pre-commit -numpy==1.22.0 +numpy==1.23.1 # via # altair + # db-dtypes # feast (setup.py) # great-expectations # pandas @@ -401,8 +410,8 @@ packaging==21.3 # via # build # dask + # db-dtypes # deprecation - # google-api-core # google-cloud-bigquery # great-expectations # pytest @@ -411,7 +420,9 @@ packaging==21.3 pandas==1.4.3 # via # altair + # db-dtypes # feast (setup.py) + # google-cloud-bigquery # great-expectations # pandavro # snowflake-connector-python @@ -431,7 +442,7 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -pip-tools==6.7.0 +pip-tools==6.8.0 # via feast (setup.py) platformdirs==2.5.2 # via virtualenv @@ -439,9 +450,9 @@ pluggy==1.0.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.4.0 +portalocker==2.5.1 # via msal-extensions -pre-commit==2.19.0 +pre-commit==2.20.0 # via feast (setup.py) prompt-toolkit==3.0.30 # via ipython @@ -462,6 +473,7 @@ protobuf==3.20.1 # google-cloud-firestore # googleapis-common-protos # grpcio-reflection + # grpcio-status # grpcio-testing # grpcio-tools # mypy-protobuf @@ -485,7 +497,9 @@ py4j==0.10.9.5 # via pyspark pyarrow==6.0.1 # via + # db-dtypes # feast (setup.py) + # google-cloud-bigquery # snowflake-connector-python pyasn1==0.4.8 # via @@ -568,7 +582,6 @@ python-dotenv==0.20.0 pytz==2022.1 # via # babel - # google-api-core # great-expectations # moto # pandas @@ -584,7 +597,7 @@ pyyaml==6.0 # uvicorn redis==4.2.2 # via feast (setup.py) -regex==2022.6.2 +regex==2022.7.9 # via black requests==2.28.1 # via @@ -629,11 +642,8 @@ six==1.16.0 # via # azure-core # azure-identity - # google-api-core # google-auth # google-auth-httplib2 - # google-cloud-core - # google-resumable-media # grpcio # happybase # mock @@ -697,7 +707,7 @@ tomli==2.0.1 # mypy # pep517 # pytest -toolz==0.11.2 +toolz==0.12.0 # via # altair # dask @@ -712,7 +722,7 @@ traitlets==5.3.0 # jupyter-core # matplotlib-inline # nbformat -trino==0.313.0 +trino==0.314.0 # via feast (setup.py) typed-ast==1.5.4 # via black @@ -726,19 +736,19 @@ types-python-dateutil==2.8.18 # via feast (setup.py) types-pytz==2022.1.1 # via feast (setup.py) -types-pyyaml==6.0.9 +types-pyyaml==6.0.10 # via feast (setup.py) -types-redis==4.3.3 +types-redis==4.3.4 # via feast (setup.py) -types-requests==2.28.0 +types-requests==2.28.2 # via feast (setup.py) -types-setuptools==57.4.18 +types-setuptools==63.2.0 # via feast (setup.py) types-tabulate==0.8.11 # via feast (setup.py) -types-urllib3==1.26.15 +types-urllib3==1.26.16 # via types-requests -typing-extensions==4.2.0 +typing-extensions==4.3.0 # via # aioitertools # azure-core @@ -753,7 +763,7 @@ tzlocal==4.2 # via great-expectations uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.9 +urllib3==1.26.10 # via # botocore # feast (setup.py) @@ -790,7 +800,7 @@ xmltodict==0.13.0 # via moto yarl==1.7.2 # via aiohttp -zipp==3.8.0 +zipp==3.8.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 76e2815ed5..91e6602083 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=sdk/python/requirements/py3.9-requirements.txt # -absl-py==1.1.0 +absl-py==1.2.0 # via tensorflow-metadata anyio==3.6.1 # via @@ -38,7 +38,7 @@ dask==2022.1.1 # via feast (setup.py) dill==0.3.5.1 # via feast (setup.py) -fastapi==0.78.0 +fastapi==0.79.0 # via feast (setup.py) fastavro==1.5.2 # via @@ -50,9 +50,9 @@ fsspec==2022.5.0 # via dask google-api-core==2.8.2 # via feast (setup.py) -google-auth==2.9.0 +google-auth==2.9.1 # via google-api-core -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # feast (setup.py) # google-api-core @@ -73,7 +73,7 @@ idna==3.3 # requests jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.6.1 +jsonschema==4.7.2 # via feast (setup.py) locket==1.0.0 # via partd @@ -87,7 +87,7 @@ mypy==0.961 # via sqlalchemy mypy-extensions==0.4.3 # via mypy -numpy==1.22.0 +numpy==1.23.1 # via # feast (setup.py) # pandas @@ -170,7 +170,7 @@ toml==0.10.2 # via feast (setup.py) tomli==2.0.1 # via mypy -toolz==0.11.2 +toolz==0.12.0 # via # dask # partd @@ -178,13 +178,13 @@ tqdm==4.64.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -typing-extensions==4.2.0 +typing-extensions==4.3.0 # via # mypy # pydantic # sqlalchemy2-stubs # starlette -urllib3==1.26.9 +urllib3==1.26.10 # via requests uvicorn[standard]==0.18.2 # via feast (setup.py) diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 83bc1ef308..0f41176bd1 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -13,6 +13,7 @@ BigQueryLoggingDestination, SavedDatasetBigQueryStorage, ) +from feast.utils import make_df_tzaware from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) @@ -74,6 +75,10 @@ def create_data_source( f"{self.gcp_project}.{self.project_name}.{destination_name}" ) + # Make all datetime columns timezone aware. This should be the behaviour of + # `BigQueryOfflineStore.offline_write_batch`, but since we're bypassing that API here, we should follow the same + # rule. The schema of this initial dataframe determines the schema in the newly created BigQuery table. + df = make_df_tzaware(df) job = self.client.load_table_from_dataframe(df, destination_name) job.result() diff --git a/setup.py b/setup.py index deaf47f955..bbcd6ad3ec 100644 --- a/setup.py +++ b/setup.py @@ -80,11 +80,10 @@ ] GCP_REQUIRED = [ - "google-cloud-bigquery>=2,<3", + "google-cloud-bigquery[pandas]>=2,<4", "google-cloud-bigquery-storage >= 2.0.0,<3", "google-cloud-datastore>=2.1.*,<3", - "google-cloud-storage>=1.34.*,<1.41", - "google-cloud-core>=1.4.0,<2.0.0", + "google-cloud-storage>=1.34.*,<3", ] REDIS_REQUIRED = [ @@ -154,7 +153,7 @@ "sphinx-rtd-theme", "testcontainers>=3.5,<4", "adlfs==0.5.9", - "firebase-admin==4.5.2", + "firebase-admin>=5.2.0,<6", "pre-commit", "assertpy==1.1", "pip-tools",