Upload execution files to temporary storage. Closes #1349 (#1649)

* Upload execution files to temporary storage. Closes #1349

* Use scratch volume for execution file upload

* Reduced duplication of "Temporary"

* WIP add travis_wait to extend timeout in tests

Co-authored-by: Umesh Timalsina <umesh.timalsina@vanderbilt.edu>

.travis.yml

@@ -7,7 +7,7 @@ addons:
 sudo: false
 env:
     - DEEPFORGE_HOST=127.0.0.1:8080 NODE_ENV=test
-script: npm run test-ci
+script: travis_wait 30 npm run test-ci
 before_install:
   - docker pull minio/minio
   - docker run -d -p 9000:9000 --name minio-test -e "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}" -e "MINIO_SECRET_KEY=${MINIO_SECRET_KEY}" minio/minio server ./data

src/common/compute/backends/sciserver-compute/Client.js

@@ -26,7 +26,6 @@ define([
         ComputeClient.apply(this, arguments);
         this.username = config.username;
         this.password = config.password;
-        this.volume = config.volume;
         this.computeDomain = config.computeDomain;
         this.previousJobState = {};
         this.consoleOutputLen = {};
@@ -35,8 +34,8 @@ define([
     SciServerClient.prototype = Object.create(ComputeClient.prototype);
 
     SciServerClient.prototype.createJob = async function(hash) {
-        const dirname = await this._uploadFiles(hash);
-        const job = await this._createJob(dirname);
+        const filesInfo = await this._uploadFiles(hash);
+        const job = await this._createJob(filesInfo);
         const jobInfo = {
             id: job.id,
             hash,
@@ -46,13 +45,14 @@ define([
         return jobInfo;
     };
 
-    SciServerClient.prototype._createConfig = async function(dirname) {
+    SciServerClient.prototype._createConfig = async function(filesInfo) {
+        const {dirname, volumePool, volume} = filesInfo;
         const domain = await this._getComputeDomain();
         const userVolumes = domain.userVolumes.map(volume => ({
             userVolumeId: volume.id,
             needsWriteAccess: SciServerClient.isWritable(volume),
         }));
-        const filepath = `/home/idies/workspace/Storage/${dirname}`;
+        const filepath = `/home/idies/workspace/${volumePool}/${volume}/${dirname}`;
 
         return {
             command: `bash ${filepath}/prepare-and-run.sh ${filepath}`,
@@ -71,8 +71,8 @@ define([
         const config =  {
             username: this.username,
             password: this.password,
-            volume: this.volume,
-            volumePool: 'Storage'
+            volume: `${this.username}/scratch`,
+            volumePool: 'Temporary'
         };
         const storage = await Storage.getClient('sciserver-files', this.logger, config);
         const files = Object.entries(metadata.content)
@@ -85,11 +85,13 @@ define([
 
         await storage.putFile(`${dirname}/prepare-and-run.sh`, PREPARE_AND_RUN);
         await Promise.all(files);
-        return `${config.volume}/${dirname}`;
+        const filesInfo = Object.assign({}, config);
+        filesInfo.dirname = dirname;
+        return filesInfo;
     };
 
-    SciServerClient.prototype._createJob = async function(dirname) {
-        const config = await this._createConfig(dirname);
+    SciServerClient.prototype._createJob = async function(filesInfo) {
+        const config = await this._createConfig(filesInfo);
         const url = 'https://apps.sciserver.org/racm//jobm/rest/jobs/docker';
 
         const opts = {

src/common/compute/backends/sciserver-compute/metadata.json

@@ -21,14 +21,6 @@
                 "isPassword": true
             }
         },
-        {
-            "name": "volume",
-            "displayName": "Volume",
-            "description": "Volume to use for upload.",
-            "value": "USERNAME/deepforge_data",
-            "valueType": "string",
-            "readOnly": false
-        },
         {
             "name": "computeDomain",
             "displayName": "Compute Domain",