[HP-1539] Add AWS functionality to cirrus

.pre-commit-config.yaml

@@ -13,6 +13,6 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, develop, --branch, master, --branch, main, --pattern, release/.*]
 -   repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
     -   id: black

README.md

@@ -24,6 +24,23 @@ using the library like above.
 
 So... you should at least read how to set up your environment.
 
+For AWS functionality you can use an example like
+
+```
+import boto3
+from gen3cirrus import AwsService
+
+client = boto3.client()
+
+aws = AwsService(client)
+
+object = "test.txt"
+bucket = "testBucket"
+expiration = 3600
+
+url = aws.requestorPaysDownloadPresignedURL( bucket, object, expiration)
+```
+
 ## Setting up Environment for `cirrus`
 `cirrus`'s wispy clouds must dwell in the great blue expanse with other Clouds.
 Thus, you'll need to configure `cirrus` with necessary information about those Clouds
@@ -185,6 +202,14 @@ cirrus_config.update(**settings)
 
 *Still uses Google libraries for auth*
 
+## AWS Specific Implentation Details
+
+### Method for communication with AWS's API(s)
+
+For AWS you must bring your own Boto3 client that you have configured.
+
+You can then setup the AWS service and your client will be passed as an argument to the AWS API.
+
 ## Building the Documentation
 - `pipenv install --dev`
 - `pipenv run python docs/create_docs.py`

gen3cirrus/__init__.py

@@ -1,2 +1,4 @@
 # Expose public API from each cloud provider
 from .google_cloud import GoogleCloudManager
+
+from .aws import AwsCloudManager, AwsService

gen3cirrus/aws/__init__.py

@@ -0,0 +1,2 @@
+from .manager import AwsCloudManager
+from .services import AwsService

gen3cirrus/aws/manager.py

@@ -0,0 +1,7 @@
+class AwsCloudManager:
+    """
+    Manager for AWS cloud clients. Currently this is not in use but has been added in case it is needed in the future
+    """
+
+    def __init__(self):
+        pass

gen3cirrus/aws/services.py

@@ -0,0 +1,52 @@
+"""
+Amazon service for interacting with APIs
+"""
+
+import backoff
+import boto3
+
+from botocore.exceptions import ClientError
+from gen3cirrus.config import config
+from gen3cirrus.aws.utils import generatePresignedURL, generatePresignedURLRequesterPays
+
+from cdislogging import get_logger
+
+logger = get_logger(__name__, log_level="info")
+
+
+class AwsService(object):
+    """
+    Generic Amazon servicing using Boto3
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    def downloadPresignedURL(self, bucket, key, expiration):
+        """
+        Wrapper function for generating a presingned url for downloading an object
+        """
+        return generatePresignedURL(self.client, "get", bucket, key, expiration)
+
+    def uploadPresignedURL(self, bucket, key, expiration):
+        """
+        Wrapper function for generating a presingned url for uploading an object
+        """
+        return generatePresignedURL(self.client, "put", bucket, key, expiration)
+
+    def multipartUploadPresignedURL(self, bucket, key, expiration, upload_id, part):
+        """
+        Wrapper function for generating a presingned url for uploading an object
+        """
+        return generateMultipartUploadUrl(
+            self.client, bucket, key, expiration, upload_id, part
+        )
+
+    def requesterPaysDownloadPresignedURL(self, bucket, key, expiration):
+        """
+        Wrapper function for generating a presingned url for downloading an object from a requester pays bucket
+        """
+        return generatePresignedURLRequesterPays(self.client, bucket, key, expiration)
+
+    def _debug(self):
+        print("This is for debugging purposes -- REMOVE WHEN DONE")

gen3cirrus/aws/utils.py

@@ -0,0 +1,110 @@
+import datetime
+import json
+
+import boto3
+from botocore.exceptions import ClientError
+
+from cdislogging import get_logger
+
+logger = get_logger(__name__, log_level="info")
+
+
+def generatePresignedURL(client, method, bucket_name, object_name, expires):
+    """
+    Function for generating a presigned url for upload or download
+
+    Args:
+        client: s3 boto client
+        method: ["get", "put"] "get" for download and "put" for upload
+        bucket_name: s3 bucket name
+        object_name: s3 bucket object key
+        expires: time for presigned url to exist (in seconds)
+    """
+
+    s3_client = client
+
+    if method == "get":
+        m = "get_object"
+    elif method == "put":
+        m = "put_object"
+    else:
+        logger.error(
+            "method for generating presigned url must be 'get' for download or 'put' for upload"
+        )
+        return None
+
+    try:
+        response = s3_client.generate_presigned_url(
+            m, Params={"Bucket": bucket_name, "Key": object_name}, ExpiresIn=expires
+        )
+
+    except ClientError as e:
+        logger.error(e)
+        return None
+
+    return response
+
+
+def generateMultipartUploadUrl(
+    client, bucket_name, object_name, expires, upload_id, part_no
+):
+    """
+    Function for generating a presigned url only for one part of multipart upload
+
+    Args:
+        client: s3 boto client
+        method: ["get", "put"] "get" for download and "put" for upload
+        bucket_name: s3 bucket name
+        object_name: s3 bucket object key
+        expires: time for presigned url to exist (in seconds)
+        upload_id: ID for upload to s3
+        part_no: part number of multipart upload
+    """
+    s3_client = client
+    try:
+        response = s3_client.generate_presigned_url(
+            ClientMethod="upload_part",
+            Params={
+                "Bucket": bucket_name,
+                "Key": object_name,
+                "UploadId": upload_id,
+                "PartNumber": part_no,
+            },
+            ExpiresIn=expires,
+        )
+
+    except ClientError as e:
+        logger.error(e)
+        return None
+
+    return response
+
+
+def generatePresignedURLRequesterPays(client, bucket_name, object_name, expires):
+    """
+    Function for generating a presigned url only for requester pays buckets
+
+    Args:
+        client: s3 boto client
+        method: ["get", "put"] "get" for download and "put" for upload
+        bucket_name: s3 bucket name
+        object_name: s3 bucket object key
+        expires: time for presigned url to exist (in seconds)
+    """
+    s3_client = client
+    try:
+        response = s3_client.generate_presigned_url(
+            "get_object",
+            Params={
+                "Bucket": bucket_name,
+                "Key": object_name,
+                "RequestPayer": "requester",
+            },
+            ExpiresIn=expires,
+        )
+
+    except ClientError as e:
+        logger.error(e)
+        return None
+
+    return response

gen3cirrus/core.py

@@ -4,6 +4,7 @@
 Current Capabilities:
 - Manage Google resources, policies, and access (specific Google APIs
   are abstracted through a Management class that exposes needed behavior)
+- Manage AWS resources amd access S3 APIs
 """
 
 

gen3cirrus/errors.py

@@ -1,5 +1,7 @@
 class CirrusError(Exception):
-    def __init__(self, message="There was an error within the gen3cirrus library.", *args):
+    def __init__(
+        self, message="There was an error within the gen3cirrus library.", *args
+    ):
         super(CirrusError, self).__init__(message)