diff --git a/sdk/python/tests/integration/registration/test_cli_local.py b/sdk/python/tests/integration/registration/test_cli.py similarity index 75% rename from sdk/python/tests/integration/registration/test_cli_local.py rename to sdk/python/tests/integration/registration/test_cli.py index d59d2ebf51..695c2ceaa4 100644 --- a/sdk/python/tests/integration/registration/test_cli_local.py +++ b/sdk/python/tests/integration/registration/test_cli.py @@ -1,47 +1,41 @@ import tempfile +import uuid from contextlib import contextmanager -from pathlib import Path +from pathlib import Path, PosixPath from textwrap import dedent -import assertpy import pytest - -from feast.feature_store import FeatureStore +import yaml +from assertpy import assertpy + +from feast import FeatureStore, RepoConfig +from tests.integration.feature_repos.repo_configuration import FULL_REPO_CONFIGS +from tests.integration.feature_repos.universal.data_source_creator import ( + DataSourceCreator, +) from tests.utils.cli_utils import CliRunner, get_example_repo from tests.utils.online_read_write_test import basic_rw_test @pytest.mark.integration -def test_workflow() -> None: - """ - Test running apply on a sample repo, and make sure the infra gets created. - """ +@pytest.mark.parametrize("test_repo_config", FULL_REPO_CONFIGS) +def test_universal_cli(test_repo_config) -> None: + project = f"test_universal_cli_{str(uuid.uuid4()).replace('-', '')[:8]}" + runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: - # Construct an example repo in a temporary dir + with tempfile.TemporaryDirectory() as repo_dir_name: + feature_store_yaml = make_feature_store_yaml( + project, test_repo_config, repo_dir_name + ) repo_path = Path(repo_dir_name) - data_path = Path(data_dir_name) repo_config = repo_path / "feature_store.yaml" - repo_config.write_text( - dedent( - f""" - project: foo - registry: {data_path / "registry.db"} - provider: local - online_store: - path: {data_path / "online_store.db"} - offline_store: - type: bigquery - """ - ) - ) + repo_config.write_text(dedent(feature_store_yaml)) repo_example = repo_path / "example.py" repo_example.write_text(get_example_repo("example_feature_repo_1.py")) - result = runner.run(["apply"], cwd=repo_path) assertpy.assert_that(result.returncode).is_equal_to(0) @@ -65,6 +59,9 @@ def test_workflow() -> None: ) assertpy.assert_that(result.returncode).is_equal_to(0) + fs = FeatureStore(repo_path=str(repo_path)) + assertpy.assert_that(fs.list_feature_views()).is_length(3) + # entity & feature view describe commands should fail when objects don't exist result = runner.run(["entities", "describe", "foo"], cwd=repo_path) assertpy.assert_that(result.returncode).is_equal_to(1) @@ -76,7 +73,6 @@ def test_workflow() -> None: # Doing another apply should be a no op, and should not cause errors result = runner.run(["apply"], cwd=repo_path) assertpy.assert_that(result.returncode).is_equal_to(0) - basic_rw_test( FeatureStore(repo_path=str(repo_path), config=None), view_name="driver_locations", @@ -86,44 +82,29 @@ def test_workflow() -> None: assertpy.assert_that(result.returncode).is_equal_to(0) -@pytest.mark.integration -def test_non_local_feature_repo() -> None: - """ - Test running apply on a sample repo, and make sure the infra gets created. - """ - runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name: - - # Construct an example repo in a temporary dir - repo_path = Path(repo_dir_name) - - repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( - """ - project: foo - registry: data/registry.db - provider: local - online_store: - path: data/online_store.db - offline_store: - type: bigquery - """ - ) - ) - - repo_example = repo_path / "example.py" - repo_example.write_text(get_example_repo("example_feature_repo_1.py")) - - result = runner.run(["apply"], cwd=repo_path) - assertpy.assert_that(result.returncode).is_equal_to(0) - - fs = FeatureStore(repo_path=str(repo_path)) - assertpy.assert_that(fs.list_feature_views()).is_length(3) - - result = runner.run(["teardown"], cwd=repo_path) - assertpy.assert_that(result.returncode).is_equal_to(0) +def make_feature_store_yaml(project, test_repo_config, repo_dir_name: PosixPath): + offline_creator: DataSourceCreator = test_repo_config.offline_store_creator(project) + + offline_store_config = offline_creator.create_offline_store_config() + online_store = test_repo_config.online_store + + config = RepoConfig( + registry=str(Path(repo_dir_name) / "registry.db"), + project=project, + provider=test_repo_config.provider, + offline_store=offline_store_config, + online_store=online_store, + repo_path=str(Path(repo_dir_name)), + ) + config_dict = config.dict() + if ( + isinstance(config_dict["online_store"], dict) + and "redis_type" in config_dict["online_store"] + ): + del config_dict["online_store"]["redis_type"] + config_dict["repo_path"] = str(config_dict["repo_path"]) + + return yaml.safe_dump(config_dict) @contextmanager diff --git a/sdk/python/tests/integration/registration/test_cli_aws.py b/sdk/python/tests/integration/registration/test_cli_aws.py deleted file mode 100644 index df70b17eba..0000000000 --- a/sdk/python/tests/integration/registration/test_cli_aws.py +++ /dev/null @@ -1,64 +0,0 @@ -import random -import string -import tempfile -from pathlib import Path -from textwrap import dedent - -import pytest - -from feast.feature_store import FeatureStore -from tests.utils.cli_utils import CliRunner, get_example_repo -from tests.utils.online_read_write_test import basic_rw_test - - -@pytest.mark.integration -def test_basic() -> None: - project_id = "".join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(10) - ) - runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: - - repo_path = Path(repo_dir_name) - data_path = Path(data_dir_name) - - repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( - f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: aws - online_store: - type: dynamodb - region: us-west-2 - offline_store: - type: redshift - cluster_id: feast-integration-tests - region: us-west-2 - user: admin - database: feast - s3_staging_location: s3://feast-integration-tests/redshift - iam_role: arn:aws:iam::402087665549:role/redshift_s3_access_role - """ - ) - ) - - repo_example = repo_path / "example.py" - repo_example.write_text(get_example_repo("example_feature_repo_1.py")) - - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - # Doing another apply should be a no op, and should not cause errors - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - basic_rw_test( - FeatureStore(repo_path=str(repo_path), config=None), - view_name="driver_locations", - ) - - result = runner.run(["teardown"], cwd=repo_path) - assert result.returncode == 0 diff --git a/sdk/python/tests/integration/registration/test_cli_gcp.py b/sdk/python/tests/integration/registration/test_cli_gcp.py deleted file mode 100644 index b4be581088..0000000000 --- a/sdk/python/tests/integration/registration/test_cli_gcp.py +++ /dev/null @@ -1,86 +0,0 @@ -import random -import string -import tempfile -from pathlib import Path -from textwrap import dedent - -import pytest - -from feast.feature_store import FeatureStore -from tests.utils.cli_utils import CliRunner, get_example_repo -from tests.utils.online_read_write_test import basic_rw_test - - -@pytest.mark.integration -def test_basic() -> None: - project_id = "".join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(10) - ) - runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: - - repo_path = Path(repo_dir_name) - data_path = Path(data_dir_name) - - repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( - f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: gcp - """ - ) - ) - - repo_example = repo_path / "example.py" - repo_example.write_text(get_example_repo("example_feature_repo_1.py")) - - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - # Doing another apply should be a no op, and should not cause errors - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - basic_rw_test( - FeatureStore(repo_path=str(repo_path), config=None), - view_name="driver_locations", - ) - - result = runner.run(["teardown"], cwd=repo_path) - assert result.returncode == 0 - - -@pytest.mark.integration -def test_missing_bq_source_fail() -> None: - project_id = "".join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(10) - ) - runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: - - repo_path = Path(repo_dir_name) - data_path = Path(data_dir_name) - - repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( - f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: gcp - """ - ) - ) - - repo_example = repo_path / "example.py" - repo_example.write_text( - get_example_repo("example_feature_repo_with_missing_bq_source.py") - ) - - returncode, output = runner.run_with_output(["apply"], cwd=repo_path) - assert returncode == 1 - assert b"DataSourceNotFoundException" in output diff --git a/sdk/python/tests/integration/registration/test_cli_redis.py b/sdk/python/tests/integration/registration/test_cli_redis.py deleted file mode 100644 index a4b146a29c..0000000000 --- a/sdk/python/tests/integration/registration/test_cli_redis.py +++ /dev/null @@ -1,101 +0,0 @@ -import random -import string -import tempfile -from pathlib import Path -from textwrap import dedent - -import pytest -import redis - -from feast.feature_store import FeatureStore -from tests.utils.cli_utils import CliRunner, get_example_repo -from tests.utils.online_read_write_test import basic_rw_test - - -@pytest.mark.integration -def test_basic() -> None: - project_id = "".join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(10) - ) - runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: - - repo_path = Path(repo_dir_name) - data_path = Path(data_dir_name) - - repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( - f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: local - offline_store: - type: bigquery - online_store: - type: redis - connection_string: localhost:6379,db=0 - """ - ) - ) - - repo_example = repo_path / "example.py" - repo_example.write_text(get_example_repo("example_feature_repo_1.py")) - - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - # Doing another apply should be a no op, and should not cause errors - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - basic_rw_test( - FeatureStore(repo_path=str(repo_path), config=None), - view_name="driver_locations", - ) - - result = runner.run(["teardown"], cwd=repo_path) - assert result.returncode == 0 - - -@pytest.mark.integration -def test_connection_error() -> None: - project_id = "".join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(10) - ) - runner = CliRunner() - with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: - - repo_path = Path(repo_dir_name) - data_path = Path(data_dir_name) - - repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( - f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: local - offline_store: - type: file - online_store: - type: redis - connection_string: localhost:6379,db=0= - """ - ) - ) - - repo_example = repo_path / "example.py" - repo_example.write_text(get_example_repo("example_feature_repo_2.py")) - - result = runner.run(["apply"], cwd=repo_path) - assert result.returncode == 0 - - # Redis does not support names for its databases. - with pytest.raises(redis.exceptions.ResponseError): - basic_rw_test( - FeatureStore(repo_path=str(repo_path), config=None), - view_name="driver_hourly_stats", - )