diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 1a914473c1..4c758d7601 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -84,16 +84,6 @@ jobs: env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} - services: - redis: - image: redis - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - uses: actions/checkout@v2 with: diff --git a/.gitignore b/.gitignore index e4becb3c31..97a9d2c975 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ vendor .terraform/ *.tfvars -# python +# python # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -107,6 +107,7 @@ coverage.xml *.cover .hypothesis/ .pytest_cache/ +infra/scripts/*.conf # Translations *.mo diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index d32740adf3..6225f2d1d1 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -41,7 +41,7 @@ try: from redis import Redis - from redis.cluster import ClusterNode, RedisCluster + from rediscluster import RedisCluster except ImportError as e: from feast.errors import FeastExtrasDependencyImportError @@ -163,9 +163,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): online_store_config.connection_string ) if online_store_config.redis_type == RedisType.redis_cluster: - kwargs["startup_nodes"] = [ - ClusterNode(**node) for node in startup_nodes - ] + kwargs["startup_nodes"] = startup_nodes self._client = RedisCluster(**kwargs) else: kwargs["host"] = startup_nodes[0]["host"] diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 1b953a5707..d8f4eed9ce 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.7 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.7-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -143,8 +141,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -282,18 +278,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv + # via great-expectations importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 @@ -358,10 +343,12 @@ jupyterlab-pygments==0.1.2 jupyterlab-widgets==1.0.2 # via ipywidgets libcst==0.4.1 - # via google-cloud-datastore + # via + # google-cloud-bigquery-storage + # google-cloud-datastore locket==0.2.1 # via partd -markupsafe==2.1.0 +markupsafe==2.0.1 # via # jinja2 # moto @@ -450,7 +437,6 @@ packaging==21.3 # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -621,7 +607,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.4 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -762,10 +752,10 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy -types-protobuf==3.19.12 + # via black +types-futures==3.3.8 + # via types-protobuf +types-protobuf==3.19.2 # via # feast (setup.py) # mypy-protobuf @@ -787,22 +777,11 @@ types-urllib3==1.26.9 # via types-requests typing-extensions==4.1.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -841,9 +820,7 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 @@ -852,7 +829,6 @@ zipp==3.7.0 # via # importlib-metadata # importlib-resources - # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 7a94294c95..5a8aa35532 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.8-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -116,6 +114,8 @@ click==8.0.3 # great-expectations # pip-tools # uvicorn +cloudpickle==2.0.0 + # via dask colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3 @@ -131,6 +131,8 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python +dask==2022.1.1 + # via feast (setup.py) debugpy==1.5.1 # via ipykernel decorator==5.1.1 @@ -139,8 +141,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -181,6 +181,7 @@ frozenlist==1.3.0 fsspec==2022.1.0 # via # adlfs + # dask # gcsfs gcsfs==2022.1.0 # via feast (setup.py) @@ -277,18 +278,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv + # via great-expectations importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 @@ -356,6 +346,8 @@ libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore +locket==0.2.1 + # via partd markupsafe==2.0.1 # via # jinja2 @@ -439,12 +431,12 @@ oscrypto==1.2.1 packaging==21.3 # via # bleach + # dask # deprecation # google-api-core # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -459,6 +451,8 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi +partd==1.2.0 + # via dask pathspec==0.9.0 # via black pbr==5.8.0 @@ -604,6 +598,7 @@ pytz-deprecation-shim==0.1.0.post0 # via tzlocal pyyaml==6.0 # via + # dask # feast (setup.py) # libcst # pre-commit @@ -612,7 +607,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -727,7 +726,10 @@ tomli==2.0.0 # mypy # pep517 toolz==0.11.2 - # via altair + # via + # altair + # dask + # partd tornado==6.1 # via # ipykernel @@ -751,9 +753,7 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy + # via black types-futures==3.3.8 # via types-protobuf types-protobuf==3.19.7 @@ -778,22 +778,11 @@ types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -832,9 +821,7 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 @@ -843,7 +830,6 @@ zipp==3.7.0 # via # importlib-metadata # importlib-resources - # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 1421d7e3c3..403aab70aa 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt +# pip-compile --extra=ci --output-file=requirements/py3.9-ci-requirements.txt setup.py # absl-py==1.0.0 # via tensorflow-metadata @@ -44,8 +44,6 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp attrs==21.4.0 # via # aiohttp @@ -69,10 +67,6 @@ babel==2.9.1 # via sphinx backcall==0.2.0 # via ipython -backports.zoneinfo==0.2.1 - # via - # pytz-deprecation-shim - # tzlocal black==19.10b0 # via feast (setup.py) bleach==4.1.0 @@ -116,6 +110,8 @@ click==8.0.3 # great-expectations # pip-tools # uvicorn +cloudpickle==2.0.0 + # via dask colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3 @@ -131,6 +127,8 @@ cryptography==3.3.2 # pyjwt # pyopenssl # snowflake-connector-python +dask==2022.1.1 + # via feast (setup.py) debugpy==1.5.1 # via ipykernel decorator==5.1.1 @@ -139,8 +137,6 @@ decorator==5.1.1 # ipython defusedxml==0.7.1 # via nbconvert -deprecated==1.2.13 - # via redis deprecation==2.1.0 # via testcontainers dill==0.3.4 @@ -181,6 +177,7 @@ frozenlist==1.3.0 fsspec==2022.1.0 # via # adlfs + # dask # gcsfs gcsfs==2022.1.0 # via feast (setup.py) @@ -277,20 +274,7 @@ idna==3.3 imagesize==1.3.0 # via sphinx importlib-metadata==4.2.0 - # via - # click - # flake8 - # great-expectations - # jsonschema - # moto - # pep517 - # pluggy - # pre-commit - # pytest - # redis - # virtualenv -importlib-resources==5.4.0 - # via jsonschema + # via great-expectations iniconfig==1.1.1 # via pytest ipykernel==6.7.0 @@ -356,6 +340,8 @@ libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore +locket==0.2.1 + # via partd markupsafe==2.0.1 # via # jinja2 @@ -439,12 +425,12 @@ oscrypto==1.2.1 packaging==21.3 # via # bleach + # dask # deprecation # google-api-core # google-cloud-bigquery # google-cloud-firestore # pytest - # redis # sphinx pandas==1.3.5 # via @@ -459,6 +445,8 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi +partd==1.2.0 + # via dask pathspec==0.9.0 # via black pbr==5.8.0 @@ -604,6 +592,7 @@ pytz-deprecation-shim==0.1.0.post0 # via tzlocal pyyaml==6.0 # via + # dask # feast (setup.py) # libcst # pre-commit @@ -612,7 +601,11 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -redis==4.1.2 +redis==3.5.3 + # via + # feast (setup.py) + # redis-py-cluster +redis-py-cluster==2.1.3 # via feast (setup.py) regex==2022.1.18 # via black @@ -644,10 +637,10 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -ruamel.yaml.clib==0.2.6 - # via ruamel.yaml +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml s3transfer==0.5.0 # via boto3 scipy==1.7.3 @@ -727,7 +720,10 @@ tomli==2.0.0 # mypy # pep517 toolz==0.11.2 - # via altair + # via + # altair + # dask + # partd tornado==6.1 # via # ipykernel @@ -751,9 +747,7 @@ traitlets==5.1.1 # nbformat # notebook typed-ast==1.5.2 - # via - # black - # mypy + # via black types-futures==3.3.8 # via types-protobuf types-protobuf==3.19.7 @@ -778,22 +772,11 @@ types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via - # aiohttp - # anyio - # argon2-cffi - # asgiref - # async-timeout # great-expectations - # h11 - # importlib-metadata - # jsonschema # libcst # mypy # pydantic - # starlette # typing-inspect - # uvicorn - # yarl typing-inspect==0.7.1 # via libcst tzdata==2021.5 @@ -832,18 +815,13 @@ wheel==0.37.1 widgetsnbextension==3.5.2 # via ipywidgets wrapt==1.13.3 - # via - # deprecated - # testcontainers + # via testcontainers xmltodict==0.12.0 # via moto yarl==1.7.2 # via aiohttp zipp==3.7.0 - # via - # importlib-metadata - # importlib-resources - # pep517 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 337b437800..80daff00eb 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -78,7 +78,8 @@ ] REDIS_REQUIRED = [ - "redis>=4.1.0", + "redis==3.5.3", + "redis-py-cluster>=2.1.3", "hiredis>=2.0.0", ] diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index a342884468..9de146ac12 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -30,6 +30,7 @@ ) from tests.integration.feature_repos.repo_configuration import ( FULL_REPO_CONFIGS, + REDIS_CLUSTER_CONFIG, REDIS_CONFIG, Environment, TestData, @@ -169,10 +170,14 @@ def cleanup(): return e -@pytest.fixture() +@pytest.fixture( + params=[REDIS_CONFIG, REDIS_CLUSTER_CONFIG], + scope="session", + ids=[str(c) for c in [REDIS_CONFIG, REDIS_CLUSTER_CONFIG]], +) def local_redis_environment(request, worker_id): e = construct_test_environment( - IntegrationTestRepoConfig(online_store=REDIS_CONFIG), worker_id=worker_id + IntegrationTestRepoConfig(online_store=request.param), worker_id=worker_id ) def cleanup(): @@ -192,6 +197,16 @@ def cleanup(): return construct_universal_test_data(environment) +@pytest.fixture(scope="session") +def redis_universal_data_sources(request, local_redis_environment): + def cleanup(): + # logger.info("Running cleanup in %s, Request: %s", worker_id, request.param) + local_redis_environment.data_source_creator.teardown() + + request.addfinalizer(cleanup) + return construct_universal_test_data(local_redis_environment) + + @pytest.fixture(scope="session") def e2e_data_sources(environment: Environment, request): df = create_dataset() diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index 83fc3e03f0..89aea727a6 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -47,7 +47,7 @@ DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"} # Port 12345 will chosen as default for redis node configuration because Redis Cluster is started off of nodes 6379 -> 6384. This causes conflicts in cli integration tests so we manually keep them separate. -REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"} +REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:12345,db=0"} REDIS_CLUSTER_CONFIG = { "type": "redis", "redis_type": "redis_cluster", @@ -72,6 +72,7 @@ [ # Redis configurations IntegrationTestRepoConfig(online_store=REDIS_CONFIG), + IntegrationTestRepoConfig(online_store=REDIS_CLUSTER_CONFIG), # GCP configurations IntegrationTestRepoConfig( provider="gcp", diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 8d0c251cef..90a06bb347 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -36,13 +36,13 @@ @pytest.mark.integration -def test_entity_ttl_online_store(local_redis_environment, universal_data_sources): +def test_entity_ttl_online_store(local_redis_environment, redis_universal_data_sources): if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": return fs = local_redis_environment.feature_store # setting ttl setting in online store to 1 second fs.config.online_store.key_ttl_seconds = 1 - entities, datasets, data_sources = universal_data_sources + entities, datasets, data_sources = redis_universal_data_sources driver_hourly_stats = create_driver_hourly_stats_feature_view(data_sources.driver) driver_entity = driver()