From 1592fbb80232fa4508d0019c48292f1d3f58aca1 Mon Sep 17 00:00:00 2001 From: arcabucero Date: Sun, 18 Sep 2022 00:04:21 +0300 Subject: [PATCH 1/3] Add possibility to define feature_store.yaml path with environment variable Signed-off-by: arcabucero --- sdk/python/feast/cli.py | 2 +- sdk/python/feast/feature_store.py | 2 +- sdk/python/feast/utils.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index af9aa51191..f677584fe5 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -99,7 +99,7 @@ def cli( ctx.obj["FS_YAML_FILE"] = ( Path(feature_store_yaml).absolute() if feature_store_yaml - else ctx.obj["CHDIR"] / "feature_store.yaml" + else utils.get_default_yaml_file_path(ctx.obj["CHDIR"]) ) try: level = getattr(logging, log_level.upper()) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 9350220c21..e534a5ffc9 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -160,7 +160,7 @@ def __init__( self.config = load_repo_config(self.repo_path, fs_yaml_file) else: self.config = load_repo_config( - self.repo_path, Path(self.repo_path) / "feature_store.yaml" + self.repo_path, utils.get_default_yaml_file_path(self.repo_path) ) registry_config = self.config.get_registry_config() diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 1b99934159..da8149fb34 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1,6 +1,8 @@ +import os import typing from collections import defaultdict from datetime import datetime +from pathlib import Path from typing import Dict, List, Optional, Tuple, Union import pandas as pd @@ -51,6 +53,14 @@ def maybe_local_tz(t: datetime) -> datetime: return t +def get_default_yaml_file_path(repo_path: Path) -> Path: + if "FEAST_FS_YAML_FILE_PATH" in os.environ: + yaml_path = os.environ["FEAST_FS_YAML_FILE_PATH"] + return Path(yaml_path) / "feature_store.yaml" + else: + return repo_path / "feature_store.yaml" + + def _get_requested_feature_views_to_features_dict( feature_refs: List[str], feature_views: List["FeatureView"], From f6515dd8a4f37fa3d6f36df45152bf84da77407f Mon Sep 17 00:00:00 2001 From: arcabucero Date: Sat, 24 Sep 2022 11:05:54 +0300 Subject: [PATCH 2/3] Add possibility to define feature_store.yaml path with environment variable for aws and lambda_engine Signed-off-by: arcabucero --- sdk/python/feast/constants.py | 3 +++ sdk/python/feast/infra/aws.py | 6 +++++- .../infra/materialization/aws_lambda/lambda_engine.py | 3 ++- sdk/python/feast/utils.py | 7 ++++--- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index a2fe6f15c5..574d79f416 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -23,6 +23,9 @@ # feature_store.yaml environment variable name for remote feature server FEATURE_STORE_YAML_ENV_NAME: str = "FEATURE_STORE_YAML_BASE64" +# feature_store.yaml path environment variable name +FEAST_FS_YAML_FILE_PATH_ENV_NAME: str = "FEAST_FS_YAML_FILE_PATH" + # Environment variable for registry REGISTRY_ENV_NAME: str = "REGISTRY_BASE64" diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index f334998e6b..5a045de401 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -8,6 +8,7 @@ from colorama import Fore, Style +from feast import utils from feast.constants import ( AWS_LAMBDA_FEATURE_SERVER_IMAGE, AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, @@ -113,7 +114,10 @@ def _deploy_feature_server(self, project: str, image_uri: str): if not self.repo_config.repo_path: raise RepoConfigPathDoesNotExist() - with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f: + + with open( + utils.get_default_yaml_file_path(self.repo_config.repo_path), "rb" + ) as f: config_bytes = f.read() config_base64 = base64.b64encode(config_bytes).decode() diff --git a/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py b/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py index 53a845140e..93e33d5949 100644 --- a/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py +++ b/sdk/python/feast/infra/materialization/aws_lambda/lambda_engine.py @@ -10,6 +10,7 @@ from pydantic import StrictStr from tqdm import tqdm +from feast import utils from feast.batch_feature_view import BatchFeatureView from feast.constants import FEATURE_STORE_YAML_ENV_NAME from feast.entity import Entity @@ -137,7 +138,7 @@ def __init__( ) repo_path = self.repo_config.repo_path assert repo_path - feature_store_path = repo_path / "feature_store.yaml" + feature_store_path = utils.get_default_yaml_file_path(repo_path) self.feature_store_base64 = str( base64.b64encode(bytes(feature_store_path.read_text(), "UTF-8")), "UTF-8" ) diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index da8149fb34..50b1e73c86 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -11,6 +11,7 @@ from dateutil.tz import tzlocal from pytz import utc +from feast.constants import FEAST_FS_YAML_FILE_PATH_ENV_NAME from feast.entity import Entity from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -54,9 +55,9 @@ def maybe_local_tz(t: datetime) -> datetime: def get_default_yaml_file_path(repo_path: Path) -> Path: - if "FEAST_FS_YAML_FILE_PATH" in os.environ: - yaml_path = os.environ["FEAST_FS_YAML_FILE_PATH"] - return Path(yaml_path) / "feature_store.yaml" + if FEAST_FS_YAML_FILE_PATH_ENV_NAME in os.environ: + yaml_path = os.environ[FEAST_FS_YAML_FILE_PATH_ENV_NAME] + return Path(yaml_path) else: return repo_path / "feature_store.yaml" From 5c3a6a0094305b5ba42552998088f9ef2a3f6633 Mon Sep 17 00:00:00 2001 From: arcabucero Date: Sat, 24 Sep 2022 11:07:02 +0300 Subject: [PATCH 3/3] Add test for overriding feature_store.yaml path with environment variable Signed-off-by: arcabucero --- sdk/python/tests/unit/cli/test_cli.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdk/python/tests/unit/cli/test_cli.py b/sdk/python/tests/unit/cli/test_cli.py index f55e5ffc06..25a1dfed34 100644 --- a/sdk/python/tests/unit/cli/test_cli.py +++ b/sdk/python/tests/unit/cli/test_cli.py @@ -1,7 +1,9 @@ +import os import tempfile from contextlib import contextmanager from pathlib import Path from textwrap import dedent +from unittest import mock from assertpy import assertpy @@ -85,6 +87,21 @@ def test_3rd_party_registry_store_with_fs_yaml_override() -> None: assertpy.assert_that(return_code).is_equal_to(0) +def test_3rd_party_registry_store_with_fs_yaml_override_by_env_var() -> None: + runner = CliRunner() + + fs_yaml_file = "test_fs.yaml" + with setup_third_party_registry_store_repo( + "foo.registry_store.FooRegistryStore", fs_yaml_file_name=fs_yaml_file + ) as repo_path: + custom_yaml_path = os.path.join(repo_path, fs_yaml_file) + with mock.patch.dict( + "os.environ", {"FEAST_FS_YAML_FILE_PATH": custom_yaml_path}, clear=True + ): + return_code, output = runner.run_with_output(["apply"], cwd=repo_path) + assertpy.assert_that(return_code).is_equal_to(0) + + @contextmanager def setup_third_party_provider_repo(provider_name: str): with tempfile.TemporaryDirectory() as repo_dir_name: