Plugin repo universal tests (#1946)

* Add universal pytest marker

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Allow plugin repos to overwrite FULL_REPO_CONFIGS through environment variable

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

Makefile

@@ -61,6 +61,9 @@ test-python:
 test-python-integration:
 	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
 
+test-python-universal:
+	FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests
+
 format-python:
 	# Sort
 	cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/

sdk/python/feast/constants.py

@@ -24,3 +24,6 @@
 
 # Environment variable for toggling usage
 FEAST_USAGE = "FEAST_USAGE"
+
+# Environment variable for the path for overwriting universal test configs
+FULL_REPO_CONFIGS_MODULE_ENV_NAME: str = "FULL_REPO_CONFIGS_MODULE"

sdk/python/tests/conftest.py

@@ -40,6 +40,9 @@ def pytest_configure(config):
         "markers", "integration: mark test that has external dependencies"
     )
     config.addinivalue_line("markers", "benchmark: mark benchmarking tests")
+    config.addinivalue_line(
+        "markers", "universal: mark tests that use the universal feature repo"
+    )
 
 
 def pytest_addoption(parser):
@@ -52,11 +55,15 @@ def pytest_addoption(parser):
     parser.addoption(
         "--benchmark", action="store_true", default=False, help="Run benchmark tests",
     )
+    parser.addoption(
+        "--universal", action="store_true", default=False, help="Run universal tests",
+    )
 
 
 def pytest_collection_modifyitems(config, items: List[Item]):
     should_run_integration = config.getoption("--integration") is True
     should_run_benchmark = config.getoption("--benchmark") is True
+    should_run_universal = config.getoption("--universal") is True
 
     integration_tests = [t for t in items if "integration" in t.keywords]
     if not should_run_integration:
@@ -76,6 +83,12 @@ def pytest_collection_modifyitems(config, items: List[Item]):
         for t in benchmark_tests:
             items.append(t)
 
+    universal_tests = [t for t in items if "universal" in t.keywords]
+    if should_run_universal:
+        items.clear()
+        for t in universal_tests:
+            items.append(t)
+
 
 @pytest.fixture
 def simple_dataset_1() -> pd.DataFrame:

sdk/python/tests/integration/e2e/test_universal_e2e.py

@@ -12,6 +12,7 @@
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 @pytest.mark.parametrize("infer_features", [True, False])
 def test_e2e_consistency(environment, e2e_data_sources, infer_features):
     fs = environment.feature_store

sdk/python/tests/integration/feature_repos/repo_configuration.py

@@ -1,3 +1,5 @@
+import importlib
+import os
 import tempfile
 import uuid
 from contextlib import contextmanager
@@ -9,6 +11,7 @@
 import pandas as pd
 
 from feast import FeatureStore, FeatureView, RepoConfig, driver_test_data
+from feast.constants import FULL_REPO_CONFIGS_MODULE_ENV_NAME
 from feast.data_source import DataSource
 from tests.integration.feature_repos.universal.data_source_creator import (
     DataSourceCreator,
@@ -61,7 +64,15 @@ def __repr__(self) -> str:
 
 DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"}
 REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"}
-FULL_REPO_CONFIGS: List[IntegrationTestRepoConfig] = [
+
+# FULL_REPO_CONFIGS contains the repo configurations (e.g. provider, offline store,
+# online store, test data, and more parameters) that most integration tests will test
+# against. By default, FULL_REPO_CONFIGS uses the three providers (local, GCP, and AWS)
+# with their default offline and online stores; it also tests the providers with the
+# Redis online store. It can be overwritten by specifying a Python module through the
+# FULL_REPO_CONFIGS_MODULE_ENV_NAME environment variable. In this case, that Python
+# module will be imported and FULL_REPO_CONFIGS will be extracted from the file.
+DEFAULT_FULL_REPO_CONFIGS: List[IntegrationTestRepoConfig] = [
     # Local configurations
     IntegrationTestRepoConfig(),
     IntegrationTestRepoConfig(online_store=REDIS_CONFIG),
@@ -88,6 +99,17 @@ def __repr__(self) -> str:
         online_store=REDIS_CONFIG,
     ),
 ]
+full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME)
+if full_repo_configs_module is not None:
+    try:
+        module = importlib.import_module(full_repo_configs_module)
+        FULL_REPO_CONFIGS = getattr(module, "FULL_REPO_CONFIGS")
+    except Exception:
+        pass
+    finally:
+        FULL_REPO_CONFIGS = DEFAULT_FULL_REPO_CONFIGS
+else:
+    FULL_REPO_CONFIGS = DEFAULT_FULL_REPO_CONFIGS
 
 
 def construct_universal_entities() -> Dict[str, List[Any]]:

sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py

@@ -231,6 +231,7 @@ def get_expected_training_df(
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 @pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: str(v))
 def test_historical_features(environment, universal_data_sources, full_feature_names):
     store = environment.feature_store

sdk/python/tests/integration/online_store/test_universal_online.py

@@ -22,6 +22,7 @@
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 @pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: str(v))
 def test_online_retrieval(environment, universal_data_sources, full_feature_names):
 

sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py

@@ -11,6 +11,7 @@
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 @pytest.mark.parametrize("infer_features", [True, False], ids=lambda v: str(v))
 def test_infer_odfv_features(environment, universal_data_sources, infer_features):
     store = environment.feature_store
@@ -33,6 +34,7 @@ def test_infer_odfv_features(environment, universal_data_sources, infer_features
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 def test_infer_odfv_features_with_error(environment, universal_data_sources):
     store = environment.feature_store
 

sdk/python/tests/integration/registration/test_universal_types.py

@@ -116,6 +116,7 @@ def get_fixtures(request):
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 def test_entity_inference_types_match(offline_types_test_fixtures):
     environment, config, data_source, fv = offline_types_test_fixtures
     fs = environment.feature_store
@@ -139,6 +140,7 @@ def test_entity_inference_types_match(offline_types_test_fixtures):
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 def test_feature_get_historical_features_types_match(offline_types_test_fixtures):
     environment, config, data_source, fv = offline_types_test_fixtures
     fs = environment.feature_store
@@ -185,6 +187,7 @@ def test_feature_get_historical_features_types_match(offline_types_test_fixtures
 
 
 @pytest.mark.integration
+@pytest.mark.universal
 def test_feature_get_online_features_types_match(online_types_test_fixtures):
     environment, config, data_source, fv = online_types_test_fixtures
     fv = create_feature_view(