chore: Update CI-related AWS settings due to cred swap & global names…

…pace issues (#4171) * chore: Bumping CI to test new cloud creds, do not merge Signed-off-by: Jeremy Ary <jary@redhat.com> * chore: Update S3 CI bucket to avoid global namespace issues Signed-off-by: Jeremy Ary <jary@redhat.com> --------- Signed-off-by: Jeremy Ary <jary@redhat.com>
feast-dev · May 7, 2024 · 67bea4c · 67bea4c
1 parent adc2939
commit 67bea4c
Show file tree

Hide file tree

Showing 9 changed files with 15 additions and 14 deletions.
diff --git a/Makefile b/Makefile
@@ -172,7 +172,7 @@ test-python-universal-athena:
 	ATHENA_DATA_SOURCE=AwsDataCatalog \
 	ATHENA_DATABASE=default \
 	ATHENA_WORKGROUP=primary \
-	ATHENA_S3_BUCKET_NAME=feast-integration-tests \
+	ATHENA_S3_BUCKET_NAME=feast-int-bucket \
  	python -m pytest -n 8 --integration \
  	 	-k "not test_go_feature_server and \
 		    not test_logged_features_validation and \

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
       <img src="docs/assets/feast_logo.png" width="550">
     </a>
 </p>
-<br />
+<br /> 
 
 [![unit-tests](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/unit_tests.yml)
 [![integration-tests-and-build](https://github.com/feast-dev/feast/actions/workflows/master_only.yml/badge.svg?branch=master&event=push)](https://github.com/feast-dev/feast/actions/workflows/master_only.yml)

diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md
@@ -130,8 +130,8 @@ The following inline policy can be used to grant Redshift necessary permissions
             "Action": "s3:*",
             "Effect": "Allow",
             "Resource": [
-                "arn:aws:s3:::feast-integration-tests",
-                "arn:aws:s3:::feast-integration-tests/*"
+                "arn:aws:s3:::feast-int-bucket",
+                "arn:aws:s3:::feast-int-bucket/*"
             ]
         }
     ],

diff --git a/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/athena_offline_store/tests/data_source.py
@@ -31,7 +31,7 @@ def __init__(self, project_name: str, *args, **kwargs):
         data_source = os.getenv("ATHENA_DATA_SOURCE", "AwsDataCatalog")
         database = os.getenv("ATHENA_DATABASE", "default")
         workgroup = os.getenv("ATHENA_WORKGROUP", "primary")
-        bucket_name = os.getenv("ATHENA_S3_BUCKET_NAME", "feast-integration-tests")
+        bucket_name = os.getenv("ATHENA_S3_BUCKET_NAME", "feast-int-bucket")
 
         self.client = aws_utils.get_athena_data_client(region)
         self.s3 = aws_utils.get_s3_resource(region)

diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py
@@ -461,7 +461,7 @@ def construct_test_environment(
         test_repo_config.python_feature_server and test_repo_config.provider == "aws"
     ) or test_repo_config.registry_location == RegistryLocation.S3:
         aws_registry_path = os.getenv(
-            "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries"
+            "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries"
         )
         registry: Union[str, RegistryConfig] = (
             f"{aws_registry_path}/{project}/registry.db"

diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/redshift.py
@@ -30,16 +30,17 @@ def __init__(self, project_name: str, *args, **kwargs):
         self.s3 = aws_utils.get_s3_resource(os.getenv("AWS_REGION", "us-west-2"))
 
         self.offline_store_config = RedshiftOfflineStoreConfig(
-            cluster_id=os.getenv("AWS_CLUSTER_ID", "feast-integration-tests"),
+            cluster_id=os.getenv("AWS_CLUSTER_ID", "feast-int-bucket"),
             region=os.getenv("AWS_REGION", "us-west-2"),
             user=os.getenv("AWS_USER", "admin"),
             database=os.getenv("AWS_DB", "feast"),
             s3_staging_location=os.getenv(
                 "AWS_STAGING_LOCATION",
-                "s3://feast-integration-tests/redshift/tests/ingestion",
+                "s3://feast-int-bucket/redshift/tests/ingestion",
             ),
             iam_role=os.getenv(
-                "AWS_IAM_ROLE", "arn:aws:iam::402087665549:role/redshift_s3_access_role"
+                "AWS_IAM_ROLE",
+                "arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631",
             ),
             workgroup="",
         )

diff --git a/sdk/python/tests/integration/registration/test_feature_store.py b/sdk/python/tests/integration/registration/test_feature_store.py
@@ -226,7 +226,7 @@ def feature_store_with_gcs_registry():
 @pytest.fixture
 def feature_store_with_s3_registry():
     aws_registry_path = os.getenv(
-        "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries"
+        "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries"
     )
     return FeatureStore(
         config=RepoConfig(

diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py
@@ -53,7 +53,7 @@ def gcs_registry() -> Registry:
 @pytest.fixture
 def s3_registry() -> Registry:
     aws_registry_path = os.getenv(
-        "AWS_REGISTRY_PATH", "s3://feast-integration-tests/registries"
+        "AWS_REGISTRY_PATH", "s3://feast-int-bucket/registries"
     )
     registry_config = RegistryConfig(
         path=f"{aws_registry_path}/{int(time.time() * 1000)}/registry.db",

diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py
@@ -112,12 +112,12 @@ def retrieval_job(request, environment):
         return FileRetrievalJob(lambda: 1, full_feature_names=False)
     elif request.param is RedshiftRetrievalJob:
         offline_store_config = RedshiftOfflineStoreConfig(
-            cluster_id="feast-integration-tests",
+            cluster_id="feast-int-bucket",
             region="us-west-2",
             user="admin",
             database="feast",
-            s3_staging_location="s3://feast-integration-tests/redshift/tests/ingestion",
-            iam_role="arn:aws:iam::402087665549:role/redshift_s3_access_role",
+            s3_staging_location="s3://feast-int-bucket/redshift/tests/ingestion",
+            iam_role="arn:aws:iam::585132637328:role/service-role/AmazonRedshift-CommandsAccessRole-20240403T092631",
             workgroup="",
         )
         environment.test_repo_config.offline_store = offline_store_config