aws requester pays

.secrets.baseline

@@ -268,14 +268,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1569
+        "line_number": 1570
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1593
+        "line_number": 1594
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -422,5 +422,5 @@
       }
     ]
   },
-  "generated_at": "2024-07-25T17:19:58Z"
+  "generated_at": "2024-08-22T19:43:39Z"
 }

fence/blueprints/data/indexd.py

@@ -1,16 +1,18 @@
 import re
 import time
 import json
+import boto3
+from botocore.client import Config
 from urllib.parse import urlparse, ParseResult, urlunparse
 from datetime import datetime, timedelta
 
 from sqlalchemy.sql.functions import user
 from cached_property import cached_property
 import gen3cirrus
 from gen3cirrus import GoogleCloudManager
+from gen3cirrus import AwsService
 from cdislogging import get_logger
 from cdispyutils.config import get_value
-from cdispyutils.hmac4 import generate_aws_presigned_url
 import flask
 from flask import current_app
 import requests
@@ -396,7 +398,7 @@ def make_signed_url(self, file_name, protocol=None, expires_in=None, bucket=None
     @staticmethod
     def init_multipart_upload(key, expires_in=None, bucket=None):
         """
-        Initilize multipart upload given key
+        Initialize multipart upload given key
 
         Args:
             key(str): object key
@@ -441,7 +443,7 @@ def generate_aws_presigned_url_for_part(
 
         Args:
             key(str): object key of `guid/filename`
-            uploadID(str): uploadId of the current upload.
+            uploadId(str): uploadId of the current upload.
             partNumber(int): the part number
 
         Returns:
@@ -1061,6 +1063,8 @@ def get_signed_url(
         bucket_name = self.bucket_name()
         bucket = s3_buckets.get(bucket_name)
 
+        object_id = self.parsed_url.path.strip("/")
+
         if bucket and bucket.get("endpoint_url"):
             http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
                 self.parsed_url.netloc, self.parsed_url.path.strip("/")
@@ -1092,18 +1096,38 @@ def get_signed_url(
             region = flask.current_app.boto.get_bucket_region(
                 self.parsed_url.netloc, credential
             )
+        s3client = boto3.client(
+            "s3",
+            aws_access_key_id=credential["aws_access_key_id"],
+            aws_secret_access_key=credential["aws_secret_access_key"],
+            aws_session_token=credential.get("aws_session_token", None),
+            region_name=region,
+            config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"),
+        )
 
+        cirrus_aws = AwsService(s3client)
         auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user)
 
-        url = generate_aws_presigned_url(
-            http_url,
-            ACTION_DICT["s3"][action],
-            credential,
-            "s3",
-            region,
-            expires_in,
-            auth_info,
-        )
+        action = ACTION_DICT["s3"][action]
+
+        # get presigned url for upload
+        if action == "PUT":
+            url = cirrus_aws.upload_presigned_url(
+                bucket_name, object_id, expires_in, None
+            )
+        # get presigned url for download
+        else:
+            if bucket.get("requester_pays") is True:
+                # need to add extra parameter to signing url for header
+                # https://github.com/boto/boto3/issues/3685
+                auth_info["x-amz-request-payer"] = "requester"
+                url = cirrus_aws.requester_pays_download_presigned_url(
+                    bucket_name, object_id, expires_in, auth_info
+                )
+            else:
+                url = cirrus_aws.download_presigned_url(
+                    bucket_name, object_id, expires_in, auth_info
+                )
 
         return url
 
@@ -1115,7 +1139,7 @@ def init_multipart_upload(self, expires_in):
             expires(int): expiration time
 
         Returns:
-            UploadId(str)
+            uploadId(str)
         """
         aws_creds = get_value(
             config, "AWS_CREDENTIALS", InternalError("credentials not configured")
@@ -1133,18 +1157,19 @@ def generate_presigned_url_for_part_upload(self, uploadId, partNumber, expires_i
         Generate presigned url for uploading object part given uploadId and part number
 
         Args:
-            uploadId(str): uploadID of the multipart upload
+            uploadId(str): uploadId of the multipart upload
             partNumber(int): part number
             expires(int): expiration time
 
         Returns:
             presigned_url(str)
         """
+        bucket_name = self.bucket_name()
         aws_creds = get_value(
             config, "AWS_CREDENTIALS", InternalError("credentials not configured")
         )
         credential = S3IndexedFileLocation.get_credential_to_access_bucket(
-            self.bucket_name(), aws_creds, expires_in
+            bucket_name, aws_creds, expires_in
         )
 
         region = self.get_bucket_region()
@@ -1154,7 +1179,7 @@ def generate_presigned_url_for_part_upload(self, uploadId, partNumber, expires_i
             )
 
         return multipart_upload.generate_presigned_url_for_uploading_part(
-            self.parsed_url.netloc,
+            bucket_name,
             self.parsed_url.path.strip("/"),
             credential,
             uploadId,

fence/blueprints/data/multipart_upload.py

@@ -1,10 +1,11 @@
 import boto3
+from botocore.client import Config
 from botocore.exceptions import ClientError
 from retry.api import retry_call
 
-from cdispyutils.hmac4 import generate_aws_presigned_url
 from cdispyutils.config import get_value
 from cdislogging import get_logger
+from gen3cirrus import AwsService
 from fence.config import config
 from fence.errors import InternalError
 
@@ -58,7 +59,7 @@ def initialize_multipart_upload(bucket_name, key, credentials):
                 key, error
             )
         )
-        raise InternalError("Can not initilize multipart upload for {}".format(key))
+        raise InternalError("Can not initialize multipart upload for {}".format(key))
 
     return multipart_upload.get("UploadId")
 
@@ -140,28 +141,21 @@ def generate_presigned_url_for_uploading_part(
     Returns:
         presigned_url(str)
     """
-    s3_buckets = get_value(
-        config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured")
-    )
-    bucket = s3_buckets.get(bucket_name)
-
-    s3_buckets = get_value(
-        config, "S3_BUCKETS", InternalError("S3_BUCKETS not configured")
-    )
-    bucket = s3_buckets.get(bucket_name)
-
-    if bucket.get("endpoint_url"):
-        url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
-            bucket_name, key.strip("/")
+    try:
+        s3client = boto3.client(
+            "s3",
+            aws_access_key_id=credentials["aws_access_key_id"],
+            aws_secret_access_key=credentials["aws_secret_access_key"],
+            aws_session_token=credentials.get("aws_session_token", None),
+            region_name=region,
+            config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"),
         )
-    else:
-        url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key)
-    additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId}
+        cirrus_aws = AwsService(s3client)
 
-    try:
-        presigned_url = generate_aws_presigned_url(
-            url, "PUT", credentials, "s3", region, expires, additional_signed_qs
+        presigned_url = cirrus_aws.multipart_upload_presigned_url(
+            bucket_name, key, expires, uploadId, partNumber
         )
+
         return presigned_url
     except Exception as e:
         raise InternalError(

fence/config-default.yaml

@@ -676,6 +676,10 @@ S3_BUCKETS: {}
 #     cred: 'CRED1'
 #     region: 'us-east-1'
 #     role-arn: 'arn:aws:iam::role1'
+#   bucket5:
+#     cred: 'CRED3'
+#     region: 'us-east-1'
+#     requester_pays: true # to indicate this is a requester pay enabled S3 bucket
 GS_BUCKETS: {}
 # NOTE: Remove the {} and supply buckets if needed. Example in comments below
 #   bucket1: