Support prerequisites in webportal and rest-server (#5277)

microsoft · Feb 8, 2021 · 2c60538 · 2c60538
1 parent b13b507
commit 2c60538
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 27 deletions.
diff --git a/src/rest-server/src/config/v2/protocol.js b/src/rest-server/src/config/v2/protocol.js
@@ -88,12 +88,21 @@ const protocolSchema = {
                     type: 'string',
                     enum: ['script', 'output'],
                   },
+                  plugin: {
+                    type: 'string',
+                  },
+                  require: {
+                    type: 'array',
+                    items: {
+                      type: 'string',
+                    },
+                  },
                   uri: {
                     type: 'string',
                   },
                 },
-                required: ['name', 'type', 'uri'],
-                additionalProperties: false,
+                required: ['name', 'type'],
+                additionalProperties: true,
               },
             },
           },
@@ -109,15 +118,24 @@ const protocolSchema = {
                     type: 'string',
                     enum: ['data'],
                   },
+                  plugin: {
+                    type: 'string',
+                  },
+                  require: {
+                    type: 'array',
+                    items: {
+                      type: 'string',
+                    },
+                  },
                   uri: {
                     type: 'array',
                     items: {
                       type: 'string',
                     },
                   },
                 },
-                required: ['name', 'type', 'uri'],
-                additionalProperties: false,
+                required: ['name', 'type'],
+                additionalProperties: true,
               },
             },
           },
@@ -199,6 +217,12 @@ const protocolSchema = {
             taskRetryCount: {
               type: 'integer',
             },
+            prerequisites: {
+              type: 'array',
+              items: {
+                type: 'string',
+              },
+            },
             dockerImage: {
               type: 'string',
             },

diff --git a/src/rest-server/src/middlewares/v2/protocol.js b/src/rest-server/src/middlewares/v2/protocol.js
@@ -63,7 +63,9 @@ const protocolValidate = (protocolYAML) => {
     );
   }
   // convert prerequisites list to dict
+  // , and record all prerequisites in prerequisiteSet
   const prerequisites = {};
+  const prerequisiteSet = new Set();
   for (const type of prerequisiteTypes) {
     prerequisites[type] = {};
   }
@@ -82,6 +84,7 @@ const protocolValidate = (protocolYAML) => {
         );
       } else {
         prerequisites[item.type][item.name] = item;
+        prerequisiteSet.add(item.name);
       }
     }
   }
@@ -103,6 +106,20 @@ const protocolValidate = (protocolYAML) => {
   }
   protocolObj.deployments = deployments;
   // check prerequisites in taskRoles
+  for (const taskRole of Object.keys(protocolObj.taskRoles)) {
+    if ('prerequisites' in protocolObj.taskRoles[taskRole]) {
+      for (const prerequisite of protocolObj.taskRoles[taskRole]
+        .prerequisites) {
+        if (!prerequisiteSet.has(prerequisite)) {
+          throw createError(
+            'Bad Request',
+            'InvalidProtocolError',
+            `Prerequisite ${prerequisite} does not exist.`,
+          );
+        }
+      }
+    }
+  }
   for (const taskRole of Object.keys(protocolObj.taskRoles)) {
     for (const field of prerequisiteFields) {
       if (

diff --git a/src/webportal/src/app/job-submission/models/job-task-role.js b/src/webportal/src/app/job-submission/models/job-task-role.js
@@ -49,6 +49,7 @@ export class JobTaskRole {
       containerSize,
       isContainerSizeEnabled,
       taskRetryCount,
+      prerequisites,
       extraOptions,
     } = props;
     this.name = name || '';
@@ -62,6 +63,7 @@ export class JobTaskRole {
     this.containerSize = containerSize || getDefaultContainerSize();
     this.isContainerSizeEnabled = isContainerSizeEnabled || false;
     this.taskRetryCount = taskRetryCount || 0;
+    this.prerequisites = prerequisites || [];
     this.extraOptions = extraOptions || {};
   }
 
@@ -122,6 +124,7 @@ export class JobTaskRole {
     const jobTaskRole = new JobTaskRole({
       name: name,
       instances: instances,
+      prerequisites: get(taskRoleProtocol, 'prerequisites', []),
       completion: Completion.fromProtocol(completion),
       commands: isNil(commands) ? '' : commands.join('\n'),
       containerSize: resourcePerInstance,
@@ -178,6 +181,7 @@ export class JobTaskRole {
       instances: this.instances,
       completion: this.completion,
       taskRetryCount: this.taskRetryCount,
+      prerequisites: this.prerequisites,
       dockerImage: this.dockerInfo.name,
       data: this.extraOptions.data,
       output: this.extraOptions.output,

diff --git a/src/webportal/src/app/job-submission/models/protocol-schema.js b/src/webportal/src/app/job-submission/models/protocol-schema.js
@@ -46,6 +46,8 @@ export const taskRoleSchema = Joi.object().keys({
   data: Joi.string(),
   output: Joi.string(),
   script: Joi.string(),
+  // allow referencing prerequisites directly
+  prerequisites: Joi.array().items(Joi.string()),
   extraContainerOptions: Joi.object().keys({
     shmMB: Joi.number(),
     infiniband: Joi.boolean(),
@@ -69,30 +71,34 @@ export const taskRolesSchema = Joi.object().pattern(
   taskRoleSchema.required(),
 );
 
-export const prerequisitesSchema = Joi.object().keys({
-  protocolVersion: [Joi.string(), Joi.number()],
-  name: Joi.string()
-    .required()
-    .regex(/^[a-zA-Z0-9_-]+$/),
-  type: Joi.string()
-    .valid(['data', 'script', 'dockerimage', 'output'])
-    .required(),
-  contributor: Joi.string(),
-  description: Joi.string(),
-  auth: Joi.object().keys({
-    username: Joi.string(),
-    password: Joi.string(),
-    registryuri: Joi.string(),
-  }),
-  uri: Joi.when('type', {
-    is: 'data',
-    then: Joi.array()
-      .items(Joi.string())
+export const prerequisitesSchema = Joi.object()
+  .keys({
+    protocolVersion: [Joi.string(), Joi.number()],
+    name: Joi.string()
+      .required()
+      .regex(/^[a-zA-Z0-9_-]+$/),
+    type: Joi.string()
+      .valid(['data', 'script', 'dockerimage', 'output'])
       .required(),
-    otherwise: Joi.string(),
-  }).required(),
-  version: [Joi.string(), Joi.number()],
-});
+    plugin: Joi.string(),
+    require: Joi.array().items(Joi.string()),
+    contributor: Joi.string(),
+    description: Joi.string(),
+    auth: Joi.object().keys({
+      username: Joi.string(),
+      password: Joi.string(),
+      registryuri: Joi.string(),
+    }),
+    uri: Joi.when('type', {
+      is: 'data',
+      then: Joi.array()
+        .items(Joi.string())
+        .required(),
+      otherwise: Joi.string(),
+    }),
+    version: [Joi.string(), Joi.number()],
+  })
+  .unknown();
 
 const deploymentSchema = Joi.object().keys({
   name: Joi.string()