-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
TSPS-305 Add GCS service, sign PUT urls for data uploads (#116)
- Loading branch information
1 parent
d7406e2
commit 494990a
Showing
14 changed files
with
399 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 6 additions & 0 deletions
6
service/src/main/java/bio/terra/pipelines/app/configuration/external/GcsConfiguration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| package bio.terra.pipelines.app.configuration.external; | ||
|
|
||
| import org.springframework.boot.context.properties.ConfigurationProperties; | ||
|
|
||
| @ConfigurationProperties(prefix = "gcs") | ||
| public record GcsConfiguration(long signedUrlPutDurationHours, long signedUrlGetDurationHours) {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
14 changes: 14 additions & 0 deletions
14
service/src/main/java/bio/terra/pipelines/dependencies/gcs/GcsClient.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package bio.terra.pipelines.dependencies.gcs; | ||
|
|
||
| import com.google.cloud.storage.Storage; | ||
| import com.google.cloud.storage.StorageOptions; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| @Component | ||
| public class GcsClient { | ||
| public Storage getStorageService(String projectId) { | ||
| // this will use application default credentials, which is what is used by | ||
| // SamService.getTeaspoonsServiceAccountToken() | ||
| return StorageOptions.newBuilder().setProjectId(projectId).build().getService(); | ||
| } | ||
| } |
93 changes: 93 additions & 0 deletions
93
service/src/main/java/bio/terra/pipelines/dependencies/gcs/GcsService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package bio.terra.pipelines.dependencies.gcs; | ||
|
|
||
| import bio.terra.pipelines.app.configuration.external.GcsConfiguration; | ||
| import com.google.cloud.storage.BlobId; | ||
| import com.google.cloud.storage.BlobInfo; | ||
| import com.google.cloud.storage.HttpMethod; | ||
| import com.google.cloud.storage.Storage; | ||
| import com.google.cloud.storage.StorageException; | ||
| import java.net.URL; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.concurrent.TimeUnit; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.springframework.retry.support.RetryTemplate; | ||
| import org.springframework.stereotype.Service; | ||
|
|
||
| /** class to encapsulate interacting with GCS client */ | ||
| @Service | ||
| public class GcsService { | ||
|
|
||
| private final GcsClient gcsClient; | ||
| private final GcsConfiguration gcsConfiguration; | ||
| private final RetryTemplate listenerResetRetryTemplate; | ||
|
|
||
| private static final Logger logger = LoggerFactory.getLogger(GcsService.class); | ||
|
|
||
| public GcsService( | ||
| GcsClient gcsClient, | ||
| GcsConfiguration gcsConfiguration, | ||
| RetryTemplate listenerResetRetryTemplate) { | ||
| this.gcsClient = gcsClient; | ||
| this.gcsConfiguration = gcsConfiguration; | ||
| this.listenerResetRetryTemplate = listenerResetRetryTemplate; | ||
| } | ||
|
|
||
| /** | ||
| * Generates and returns a PUT (write-only) signed url for a specific object in a bucket. See | ||
| * documentation on signed urls <a | ||
| * href="https://cloud.google.com/storage/docs/access-control/signed-urls">here</a>. | ||
| * | ||
| * <p>The output URL can be used with a curl command to upload an object to the destination: `curl | ||
| * -X PUT -H 'Content-Type: application/octet-stream' --upload-file my-file '{url}'` | ||
| * | ||
| * @param projectId Google project id | ||
| * @param bucketName without a prefix | ||
| * @param objectName should include the full path of the object (subdirectories + file name) | ||
| * @return url that can be used to write an object to GCS | ||
| */ | ||
| public URL generatePutObjectSignedUrl(String projectId, String bucketName, String objectName) | ||
| throws StorageException { | ||
| // define target blob object resource | ||
| BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucketName, objectName)).build(); | ||
|
|
||
| // generate signed URL | ||
| Map<String, String> extensionHeaders = new HashMap<>(); | ||
| extensionHeaders.put("Content-Type", "application/octet-stream"); | ||
|
|
||
| URL url = | ||
| executionWithRetryTemplate( | ||
| listenerResetRetryTemplate, | ||
| () -> | ||
| gcsClient | ||
| .getStorageService(projectId) | ||
| .signUrl( | ||
| blobInfo, | ||
| gcsConfiguration.signedUrlPutDurationHours(), | ||
| TimeUnit.HOURS, | ||
| Storage.SignUrlOption.httpMethod(HttpMethod.PUT), | ||
| Storage.SignUrlOption.withExtHeaders(extensionHeaders), | ||
| Storage.SignUrlOption.withV4Signature())); | ||
|
|
||
| logger.info("Generated PUT signed URL: {}", url); | ||
|
|
||
| return url; | ||
| } | ||
|
|
||
| interface GcsAction<T> { | ||
| T execute(); | ||
| } | ||
|
|
||
| static <T> T executionWithRetryTemplate(RetryTemplate retryTemplate, GcsAction<T> action) { | ||
| return retryTemplate.execute( | ||
| context -> { | ||
| try { | ||
| return action.execute(); | ||
| } catch (StorageException e) { | ||
| // Note: GCS' StorageException contains retryable exceptions - not sure how to handle | ||
| throw new GcsServiceException("Error executing GCS action", e); | ||
| } | ||
| }); | ||
| } | ||
| } |
15 changes: 15 additions & 0 deletions
15
service/src/main/java/bio/terra/pipelines/dependencies/gcs/GcsServiceException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package bio.terra.pipelines.dependencies.gcs; | ||
|
|
||
| import bio.terra.common.exception.ErrorReportException; | ||
| import java.util.ArrayList; | ||
| import org.springframework.http.HttpStatus; | ||
|
|
||
| public class GcsServiceException extends ErrorReportException { | ||
| public GcsServiceException(String message, Throwable cause) { | ||
| super(message, cause, new ArrayList<>(), HttpStatus.INTERNAL_SERVER_ERROR); | ||
| } | ||
|
|
||
| public GcsServiceException(String message) { | ||
| super(message, new ArrayList<>(), HttpStatus.INTERNAL_SERVER_ERROR); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
19 changes: 19 additions & 0 deletions
19
service/src/test/java/bio/terra/pipelines/configuration/external/GcsConfigurationTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package bio.terra.pipelines.configuration.external; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
|
|
||
| import bio.terra.pipelines.app.configuration.external.GcsConfiguration; | ||
| import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.springframework.beans.factory.annotation.Autowired; | ||
|
|
||
| class GcsConfigurationTest extends BaseEmbeddedDbTest { | ||
| /** test reading gcs config from application yml */ | ||
| @Autowired GcsConfiguration gcsConfiguration; | ||
|
|
||
| @Test | ||
| void verifyGcsConfig() { | ||
| assertEquals(24L, gcsConfiguration.signedUrlGetDurationHours()); | ||
| assertEquals(24L, gcsConfiguration.signedUrlPutDurationHours()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 20 additions & 0 deletions
20
service/src/test/java/bio/terra/pipelines/dependencies/gcs/GcsClientTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package bio.terra.pipelines.dependencies.gcs; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertEquals; | ||
|
|
||
| import bio.terra.pipelines.testutils.BaseEmbeddedDbTest; | ||
| import com.google.cloud.storage.Storage; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.springframework.beans.factory.annotation.Autowired; | ||
|
|
||
| class GcsClientTest extends BaseEmbeddedDbTest { | ||
| @Autowired GcsClient gcsClient; | ||
|
|
||
| @Test | ||
| void getGcsStorageService() { | ||
| String projectId = "test-project-id"; | ||
| Storage storageService = gcsClient.getStorageService(projectId); | ||
|
|
||
| assertEquals(projectId, storageService.getOptions().getProjectId()); | ||
| } | ||
| } |
Oops, something went wrong.