-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Increase STS Sample Coverage (#42)
* test: Prepare for multiple samples and tests * feat: Increase STS Sample Coverage * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * style: lint * ci: Add `pre-samples-test.sh` for AWS integration testing * style: Streamline error handling Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
- Loading branch information
1 parent
94fe9de
commit 89abd10
Showing
18 changed files
with
1,170 additions
and
125 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
/** | ||
* Copyright 2022 Google LLC | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
'use strict'; | ||
|
||
async function main( | ||
projectId, | ||
description, | ||
awsSourceBucket, | ||
awsAccessKeyId, | ||
awsSecretAccessKey, | ||
gcsSinkBucket | ||
) { | ||
// [START storagetransfer_transfer_from_aws] | ||
|
||
// Imports the Google Cloud client library | ||
const { | ||
StorageTransferServiceClient, | ||
} = require('@google-cloud/storage-transfer'); | ||
|
||
/** | ||
* TODO(developer): Uncomment the following lines before running the sample. | ||
*/ | ||
// The ID of the Google Cloud Platform Project that owns the job | ||
// projectId = 'my-project-id' | ||
|
||
// A useful description for your transfer job | ||
// description = 'My transfer job' | ||
|
||
// AWS S3 source bucket name | ||
// awsSourceBucket = 'my-s3-source-bucket' | ||
|
||
// AWS Access Key ID | ||
// awsAccessKeyId = 'AKIA...' | ||
|
||
// AWS Secret Access Key | ||
// awsSecretAccessKey = 'HEAoMK2.../...ku8' | ||
|
||
// Google Cloud Storage destination bucket name | ||
// gcsSinkBucket = 'my-gcs-destination-bucket' | ||
|
||
// Creates a client | ||
const client = new StorageTransferServiceClient(); | ||
|
||
/** | ||
* Creates a one-time transfer job from Amazon S3 to Google Cloud Storage. | ||
*/ | ||
async function transferFromS3() { | ||
// Setting the start date and the end date as the same time creates a | ||
// one-time transfer | ||
const now = new Date(); | ||
const oneTimeSchedule = { | ||
day: now.getDate(), | ||
month: now.getMonth() + 1, | ||
year: now.getFullYear(), | ||
}; | ||
|
||
// Runs the request and creates the job | ||
const [transferJob] = await client.createTransferJob({ | ||
transferJob: { | ||
projectId, | ||
description, | ||
status: 'ENABLED', | ||
schedule: { | ||
scheduleStartDate: oneTimeSchedule, | ||
scheduleEndDate: oneTimeSchedule, | ||
}, | ||
transferSpec: { | ||
awsS3DataSource: { | ||
bucketName: awsSourceBucket, | ||
awsAccessKey: { | ||
accessKeyId: awsAccessKeyId, | ||
secretAccessKey: awsSecretAccessKey, | ||
}, | ||
}, | ||
gcsDataSink: { | ||
bucketName: gcsSinkBucket, | ||
}, | ||
}, | ||
}, | ||
}); | ||
|
||
console.log( | ||
`Created and ran a transfer job from '${awsSourceBucket}' to '${gcsSinkBucket}' with name ${transferJob.name}` | ||
); | ||
} | ||
|
||
transferFromS3(); | ||
// [END storagetransfer_transfer_from_aws] | ||
} | ||
|
||
main(...process.argv.slice(2)); | ||
|
||
process.on('unhandledRejection', err => { | ||
console.error(err); | ||
process.exitCode = 1; | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
/** | ||
* Copyright 2022 Google LLC | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
'use strict'; | ||
|
||
async function main(projectId, jobName) { | ||
// [START storagetransfer_get_latest_transfer_operation] | ||
|
||
// Imports the Google Cloud client library | ||
const { | ||
StorageTransferServiceClient, | ||
} = require('@google-cloud/storage-transfer'); | ||
|
||
/** | ||
* TODO(developer): Uncomment the following lines before running the sample. | ||
*/ | ||
// The ID of the Google Cloud Platform Project that owns the job | ||
// projectId = 'my-project-id' | ||
|
||
// Storage Transfer Service job name | ||
// jobName = 'transferJobs/1234567890' | ||
|
||
// Creates a client | ||
const client = new StorageTransferServiceClient(); | ||
|
||
/** | ||
* Checks the latest transfer operation for a given transfer job. | ||
*/ | ||
async function checkLatestTransferOperation() { | ||
const [transferJob] = await client.getTransferJob({projectId, jobName}); | ||
|
||
if (transferJob.latestOperationName) { | ||
const [transferOperation] = await client.operationsClient.getOperation({ | ||
name: transferJob.latestOperationName, | ||
}); | ||
|
||
const operation = JSON.stringify(transferOperation, null, 2); | ||
|
||
console.log(`Latest transfer operation for '${jobName}': ${operation}`); | ||
} else { | ||
console.log(`Transfer job '${jobName}' has not ran yet.`); | ||
} | ||
} | ||
|
||
checkLatestTransferOperation(); | ||
// [END storagetransfer_get_latest_transfer_operation] | ||
} | ||
|
||
main(...process.argv.slice(2)); | ||
|
||
process.on('unhandledRejection', err => { | ||
console.error(err); | ||
process.exitCode = 1; | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
/** | ||
* Copyright 2022 Google LLC | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
'use strict'; | ||
|
||
async function main(projectId, jobName, maxRetryDelayMillis) { | ||
// [START storagetransfer_create_retry_handler] | ||
|
||
// Imports the Google Cloud client library | ||
const { | ||
StorageTransferServiceClient, | ||
} = require('@google-cloud/storage-transfer'); | ||
|
||
/** | ||
* TODO(developer): Uncomment the following lines before running the sample. | ||
*/ | ||
// The ID of the Google Cloud Platform Project that owns the job | ||
// projectId = 'my-project-id' | ||
|
||
// Storage Transfer Service job name | ||
// jobName = 'transferJobs/1234567890' | ||
|
||
// The maximum delay time, in milliseconds, between requests | ||
// maxRetryDelayMillis = 60000 | ||
|
||
// Creates a client | ||
const client = new StorageTransferServiceClient(); | ||
|
||
/** | ||
* Check the latest transfer operation associated with a transfer job | ||
* with retries. | ||
*/ | ||
async function getTransferJobWithRetries() { | ||
// Setting the start date and the end date as the same time creates a | ||
// one-time transfer | ||
|
||
const options = { | ||
retry: { | ||
backoffSettings: { | ||
maxRetryDelayMillis, | ||
}, | ||
}, | ||
}; | ||
|
||
const [transferJob] = await client.getTransferJob( | ||
{projectId, jobName}, | ||
options | ||
); | ||
|
||
console.log( | ||
`Fetched transfer job: ${transferJob.name} with a maximum of ${maxRetryDelayMillis}ms delay time between requests` | ||
); | ||
} | ||
|
||
getTransferJobWithRetries(); | ||
// [END storagetransfer_create_retry_handler] | ||
} | ||
|
||
const [projectId, jobName, maxRetryDelayMillis] = [...process.argv.slice(2)]; | ||
|
||
main(projectId, jobName, Number.parseInt(maxRetryDelayMillis)); | ||
|
||
process.on('unhandledRejection', err => { | ||
console.error(err); | ||
process.exitCode = 1; | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
/** | ||
* Copyright 2022 Google LLC | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
'use strict'; | ||
|
||
async function main( | ||
projectId, | ||
description, | ||
gcsSourceBucket, | ||
gcsSinkBucket, | ||
startDate = new Date() | ||
) { | ||
// [START storagetransfer_transfer_to_nearline] | ||
|
||
// Imports the Google Cloud client library | ||
const { | ||
StorageTransferServiceClient, | ||
} = require('@google-cloud/storage-transfer'); | ||
|
||
/** | ||
* TODO(developer): Uncomment the following lines before running the sample. | ||
*/ | ||
// The ID of the Google Cloud Platform Project that owns the job | ||
// projectId = 'my-project-id' | ||
|
||
// A useful description for your transfer job | ||
// description = 'My transfer job' | ||
|
||
// Google Cloud Storage source bucket name | ||
// gcsSourceBucket = 'my-gcs-source-bucket' | ||
|
||
// Google Cloud Storage destination bucket name | ||
// gcsSinkBucket = 'my-gcs-destination-bucket' | ||
|
||
// Date to start daily migration | ||
// startDate = new Date() | ||
|
||
// Creates a client | ||
const client = new StorageTransferServiceClient(); | ||
|
||
/** | ||
* Create a daily migration from a GCS bucket to another GCS bucket for | ||
* objects untouched for 30+ days. | ||
*/ | ||
async function createDailyNearline30DayMigration() { | ||
// Runs the request and creates the job | ||
const [transferJob] = await client.createTransferJob({ | ||
transferJob: { | ||
projectId, | ||
description, | ||
status: 'ENABLED', | ||
schedule: { | ||
scheduleStartDate: { | ||
day: startDate.getDate(), | ||
month: startDate.getMonth() + 1, | ||
year: startDate.getFullYear(), | ||
}, | ||
}, | ||
transferSpec: { | ||
gcsDataSource: { | ||
bucketName: gcsSourceBucket, | ||
}, | ||
gcsDataSink: { | ||
bucketName: gcsSinkBucket, | ||
}, | ||
objectConditions: { | ||
minTimeElapsedSinceLastModification: { | ||
seconds: 2592000, // 30 days | ||
}, | ||
}, | ||
transferOptions: { | ||
deleteObjectsFromSourceAfterTransfer: true, | ||
}, | ||
}, | ||
}, | ||
}); | ||
|
||
console.log(`Created transferJob: ${transferJob.name}`); | ||
} | ||
|
||
createDailyNearline30DayMigration(); | ||
// [END storagetransfer_transfer_to_nearline] | ||
} | ||
|
||
const [projectId, description, gcsSourceBucket, gcsSinkBucket, startDate] = [ | ||
...process.argv.slice(2), | ||
]; | ||
|
||
main( | ||
projectId, | ||
description, | ||
gcsSourceBucket, | ||
gcsSinkBucket, | ||
new Date(startDate) | ||
); | ||
|
||
process.on('unhandledRejection', err => { | ||
console.error(err); | ||
process.exitCode = 1; | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.