aws requester pays

.secrets.baseline

@@ -268,14 +268,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1569
+        "line_number": 1573
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1593
+        "line_number": 1597
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -422,5 +422,5 @@
       }
     ]
   },
-  "generated_at": "2024-07-25T17:19:58Z"
+  "generated_at": "2024-08-13T16:48:45Z"
 }

fence/blueprints/data/indexd.py

@@ -1,16 +1,18 @@
 import re
 import time
 import json
-from urllib.parse import urlparse, ParseResult, urlunparse
+import boto3
+from botocore.client import Config
+from urllib.parse import urlparse, ParseResult, urlunparse, quote
 from datetime import datetime, timedelta
 
 from sqlalchemy.sql.functions import user
 from cached_property import cached_property
 import gen3cirrus
 from gen3cirrus import GoogleCloudManager
+from gen3cirrus import AwsService
 from cdislogging import get_logger
 from cdispyutils.config import get_value
-from cdispyutils.hmac4 import generate_aws_presigned_url
 import flask
 from flask import current_app
 import requests
@@ -1043,6 +1045,24 @@ def get_bucket_region(self):
         else:
             return bucket_cred["region"]
 
+    def is_custom(k):
+        if k == "user_id" or k == "username":
+            return True
+        else:
+            return False
+
+    def client_param_handler(*, params, context, **_kw):
+        # Store custom parameters in context for later event handlers
+        context["custom_params"] = {k: v for k, v in params.items() if is_custom(k)}
+        # Remove custom parameters from client parameters,
+        # because validation would fail on them
+        return {k: v for k, v in params.items() if not is_custom(k)}
+
+    def request_param_injector(*, request, **_kw):
+        if request.context["custom_params"]:
+            request.url += "&" if "?" in request.url else "?"
+            request.url += urlencode(request.context["custom_params"])
+
     def get_signed_url(
         self,
         action,
@@ -1061,6 +1081,8 @@ def get_signed_url(
         bucket_name = self.bucket_name()
         bucket = s3_buckets.get(bucket_name)
 
+        object_id = self.parsed_url.path.strip("/")
+
         if bucket and bucket.get("endpoint_url"):
             http_url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
                 self.parsed_url.netloc, self.parsed_url.path.strip("/")
@@ -1093,18 +1115,38 @@ def get_signed_url(
                 self.parsed_url.netloc, credential
             )
 
-        auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user)
-
-        url = generate_aws_presigned_url(
-            http_url,
-            ACTION_DICT["s3"][action],
-            credential,
+        client = boto3.client(
             "s3",
-            region,
-            expires_in,
-            auth_info,
+            aws_access_key_id=credential["aws_access_key_id"],
+            aws_secret_access_key=credential["aws_secret_access_key"],
+            region_name=region,
+            config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"),
         )
 
+        client.meta.events.register(
+            "provide-client-params.s3.GetObject", client_param_handler
+        )
+        client.meta.events.register("before-sign.s3.GetObject", request_param_injector)
+
+        cirrus_aws = AwsService(client)
+        auth_info = _get_auth_info_for_id_or_from_request(user=authorized_user)
+
+        action = ACTION_DICT["s3"][action]
+
+        if action == "PUT":  # get presigned url for upload
+            url = cirrus_aws.upload_presigned_url(
+                bucket_name, object_id, expires_in, auth_info
+            )
+        else:  # get presigned url for download
+            if bucket.get("requester_pays") == True:
+                url = cirrus_aws.requester_pays_download_presigned_url(
+                    bucket_name, object_id, expires_in, auth_info
+                )
+            else:
+                url = cirrus_aws.download_presigned_url(
+                    bucket_name, object_id, expires_in, auth_info
+                )
+
         return url
 
     def init_multipart_upload(self, expires_in):

fence/blueprints/data/multipart_upload.py

@@ -1,8 +1,8 @@
 import boto3
+from botocore.client import Config
 from botocore.exceptions import ClientError
 from retry.api import retry_call
 
-from cdispyutils.hmac4 import generate_aws_presigned_url
 from cdispyutils.config import get_value
 from cdislogging import get_logger
 from fence.config import config
@@ -150,18 +150,19 @@ def generate_presigned_url_for_uploading_part(
     )
     bucket = s3_buckets.get(bucket_name)
 
-    if bucket.get("endpoint_url"):
-        url = bucket["endpoint_url"].strip("/") + "/{}/{}".format(
-            bucket_name, key.strip("/")
+    try:
+        s3client = boto3.client(
+            "s3",
+            aws_access_key_id=credentials["aws_access_key_id"],
+            aws_secret_access_key=credentials["aws_secret_access_key"],
+            config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"),
         )
-    else:
-        url = "https://{}.s3.amazonaws.com/{}".format(bucket_name, key)
-    additional_signed_qs = {"partNumber": str(partNumber), "uploadId": uploadId}
+        cirrus_aws = AwsService(client)
 
-    try:
-        presigned_url = generate_aws_presigned_url(
-            url, "PUT", credentials, "s3", region, expires, additional_signed_qs
+        presigned_url = cirrus_aws.multipart_upload_presigned_url(
+            bucket, key, expires, uploadId, partNumber
         )
+
         return presigned_url
     except Exception as e:
         raise InternalError(