feat(samples): Add Azure Sample (#134)

storagetransfer/azure-request.js

@@ -0,0 +1,110 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+async function main(
+  projectId,
+  description,
+  azureStorageAccount,
+  azureSourceContainer,
+  gcsSinkBucket,
+  azureSASToken = process.env.AZURE_SAS_TOKEN
+) {
+  // [START storagetransfer_transfer_from_azure]
+
+  // Imports the Google Cloud client library
+  const {
+    StorageTransferServiceClient,
+  } = require('@google-cloud/storage-transfer');
+
+  /**
+   * TODO(developer): Uncomment the following lines before running the sample.
+   */
+  // The ID of the Google Cloud Platform Project that owns the job
+  // projectId = 'my-project-id'
+
+  // A useful description for your transfer job
+  // description = 'My transfer job'
+
+  // Azure Storage Account name
+  // azureStorageAccount = 'accountname'
+
+  // Azure Storage Account name
+  // azureSourceContainer = 'my-azure-source-bucket'
+
+  // Azure Shared Access Signature token
+  // azureSASToken = '?sv=...'
+
+  // Google Cloud Storage destination bucket name
+  // gcsSinkBucket = 'my-gcs-destination-bucket'
+
+  // Creates a client
+  const client = new StorageTransferServiceClient();
+
+  /**
+   * Creates a one-time transfer job from Azure Blob Storage to Google Cloud Storage.
+   */
+  async function transferFromBlobStorage() {
+    // Setting the start date and the end date as the same time creates a
+    // one-time transfer
+    const now = new Date();
+    const oneTimeSchedule = {
+      day: now.getDate(),
+      month: now.getMonth() + 1,
+      year: now.getFullYear(),
+    };
+
+    // Runs the request and creates the job
+    const [transferJob] = await client.createTransferJob({
+      transferJob: {
+        projectId,
+        description,
+        status: 'ENABLED',
+        schedule: {
+          scheduleStartDate: oneTimeSchedule,
+          scheduleEndDate: oneTimeSchedule,
+        },
+        transferSpec: {
+          azureBlobStorageDataSource: {
+            azureCredentials: {
+              sasToken: azureSASToken,
+            },
+            container: azureSourceContainer,
+            storageAccount: azureStorageAccount,
+          },
+          gcsDataSink: {
+            bucketName: gcsSinkBucket,
+          },
+        },
+      },
+    });
+
+    console.log(
+      `Created and ran a transfer job from '${azureSourceContainer}' to '${gcsSinkBucket}' with name ${transferJob.name}`
+    );
+  }
+
+  transferFromBlobStorage();
+  // [END storagetransfer_transfer_from_azure]
+}
+
+main(...process.argv.slice(2));
+
+process.on('unhandledRejection', err => {
+  console.error(err);
+  process.exitCode = 1;
+});

storagetransfer/package.json

@@ -16,6 +16,7 @@
     "@google-cloud/storage-transfer": "^2.2.1"
   },
   "devDependencies": {
+    "@azure/storage-blob": "^12.12.0",
     "@google-cloud/storage": "^6.0.0",
     "aws-sdk": "^2.1073.0",
     "c8": "^7.1.0",

storagetransfer/test/azure-request.test.js

@@ -0,0 +1,83 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const {assert} = require('chai');
+const {after, before, describe, it} = require('mocha');
+
+const {BucketManager, TransferJobManager, runSample} = require('./utils');
+
+describe('azure-request', () => {
+  const testBucketManager = new BucketManager();
+  const testTransferJobManager = new TransferJobManager();
+
+  let projectId;
+  let description;
+  let azureStorageAccount;
+  let azureSourceContainer;
+  let gcsSinkBucket;
+
+  before(async () => {
+    assert(
+      process.env.AZURE_CONNECTION_STRING,
+      'environment variable AZURE_CONNECTION_STRING is required'
+    );
+
+    testBucketManager.setupBlobStorageFromConnectionString(
+      process.env.AZURE_CONNECTION_STRING
+    );
+
+    azureStorageAccount =
+      process.env.AZURE_STORAGE_ACCOUNT ||
+      testBucketManager.blobStorage.accountName;
+
+    projectId = await testBucketManager.getProjectId();
+    azureSourceContainer =
+      await testBucketManager.generateBlobStorageContainer();
+    gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name;
+    description = `My transfer job from '${azureSourceContainer}' -> '${gcsSinkBucket}'`;
+
+    if (!process.env.AZURE_SAS_TOKEN) {
+      // For security purposes we only want to pass this value via environment, not cli
+      process.env.AZURE_SAS_TOKEN = new URL(
+        testBucketManager.blobStorage.storageClientContext.url
+      ).search;
+    }
+  });
+
+  after(async () => {
+    await testBucketManager.deleteBuckets();
+    await testTransferJobManager.cleanUp();
+  });
+
+  it('should create a transfer job from Azure to GCS', async () => {
+    const output = await runSample('azure-request', [
+      projectId,
+      description,
+      azureStorageAccount,
+      azureSourceContainer,
+      gcsSinkBucket,
+    ]);
+
+    assert.include(output, 'Created and ran a transfer job');
+
+    // If it ran successfully and a job was created, delete it to clean up
+    const [jobName] = output.match(/transferJobs.*/);
+
+    testTransferJobManager.transferJobToCleanUp(jobName);
+  });
+});

storagetransfer/test/utils/bucket.js

@@ -22,6 +22,7 @@ const {
   StorageTransferServiceClient,
 } = require('@google-cloud/storage-transfer');
 const AWS = require('aws-sdk');
+const AzureStorageBlob = require('@azure/storage-blob');
 const uuid = require('uuid');
 
 class BucketManager {
@@ -38,12 +39,21 @@ class BucketManager {
      * @type {Bucket[]}
      */
     this.gcsBuckets = [];
+    /**
+     * @type {string[]}
+     */
+    this.blobStorageContainers = [];
     /**
      * @type {string[]}
      */
     this.s3Buckets = [];
   }
 
+  setupBlobStorageFromConnectionString(connectionString = '') {
+    this.blobStorage =
+      AzureStorageBlob.BlobServiceClient.fromConnectionString(connectionString);
+  }
+
   setupS3(options = {}) {
     this.s3 = new AWS.S3({apiVersion: '2006-03-01', ...options});
   }
@@ -156,11 +166,9 @@ class BucketManager {
    * Configures STS read/write perms on the bucket.
    *
    * Is cached for easy clean-up via {#deleteBuckets}.
-   *
-   * @returns
    */
   async generateGCSBucket() {
-    const name = await BucketManager.generateBucketName();
+    const name = BucketManager.generateBucketName();
     const bucket = this.storage.bucket(name);
     this.gcsBuckets.push(bucket);
 
@@ -171,16 +179,30 @@ class BucketManager {
   }
 
   /**
-   * Generates a unique GCS bucket for testing.
-   * Configures STS read/write perms on the bucket.
+   * Generates a unique Azure container for testing.
    *
    * Is cached for easy clean-up via {#deleteBuckets}.
+   */
+  async generateBlobStorageContainer() {
+    const name = BucketManager.generateBucketName();
+
+    // Create a container
+    const containerClient = this.blobStorage.getContainerClient(name);
+    await containerClient.create();
+
+    this.blobStorageContainers.push(name);
+
+    return name;
+  }
+
+  /**
+   * Generates a unique S3 bucket for testing.
    *
-   * @returns
+   * Is cached for easy clean-up via {#deleteBuckets}.
    */
 
   async generateS3Bucket() {
-    const name = await BucketManager.generateBucketName();
+    const name = BucketManager.generateBucketName();
 
     await new Promise((resolve, reject) => {
       this.s3.createBucket({Bucket: name}, (error, data) => {
@@ -207,6 +229,14 @@ class BucketManager {
       }
     }
 
+    for (const container of this.blobStorageContainers) {
+      try {
+        await this.blobStorage.deleteContainer(container);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+
     for (const bucket of this.s3Buckets) {
       try {
         await new Promise((resolve, reject) => {