Make debugging reservation time configurable.

src/rest-server/config/rest-server.md

@@ -19,6 +19,7 @@ other config fields are optional, includes:
 - `github-owner: Microsoft` The marketplace repo owner in GitHub
 - `github-repository: pai` The marketplace repo name
 - `github-path: marketplace` The marketpalce path in the repo
+- `debugging-reservation-seconds` The seconds to reserved a job container to debug.
 
 ## Generated Configuration <a name="G_Config"></a>
 
@@ -34,6 +35,7 @@ rest-server:
     github-owner: Microsoft
     github-repository: pai
     github-path: marketplace
+    debugging-reservation-seconds: 604800
 ```
 
 ## Table <a name="T_Config"></a>
@@ -99,4 +101,10 @@ rest-server:
     <td>cluster_cfg["rest-server"]["etcd-uris"]</td>
     <td>String</td>
 </tr>
+<tr>
+    <td>rest-server.debugging-reservation-seconds</td>
+    <td>com["rest-server"]["debugging-reservation-seconds"]</td>
+    <td>cluster_cfg["rest-server"]["debugging-reservation-seconds"]</td>
+    <td>String</td>
+</tr>
 </table>

src/rest-server/config/rest-server.yaml

@@ -20,3 +20,4 @@ jwt-secret: pai-secret
 github-owner: Microsoft
 github-repository: pai
 github-path: marketplace
+debugging-reservation-seconds: 604800

src/rest-server/config/rest_server.py

@@ -29,6 +29,12 @@ def validation_pre(self):
             return False, '"default-pai-admin-username" is required in rest-server'
         if 'default-pai-admin-password' not in self.service_configuration:
             return False, '"default-pai-admin-password" is required in rest-server'
+        try:
+            reservation_time = int(self.service_configuration['debugging-reservation-seconds'])
+        except ValueError:
+            return False, '"debugging-reservation-seconds" should be a positive integer.'
+        if reservation_time <= 0:
+            return False, '"debugging-reservation-seconds" should be a positive integer.'
 
         return True, None
 
@@ -50,6 +56,7 @@ def run(self):
         service_object_model['github-owner'] = self.service_configuration['github-owner']
         service_object_model['github-repository'] = self.service_configuration['github-repository']
         service_object_model['github-path'] = self.service_configuration['github-path']
+        service_object_model['debugging-reservation-seconds'] = self.service_configuration['debugging-reservation-seconds']
 
         return service_object_model
 

src/rest-server/deploy/rest-server.yaml.template

@@ -57,6 +57,8 @@ spec:
           value: {{ cluster_cfg['layout']['kubernetes']['api-servers-url'] }}
         - name: AZ_RDMA
           value: "{{ cluster_cfg['cluster']['common']['az-rdma']}}"
+        - name: DEBUGGING_RESERVATION_SECONDS
+          value: "{{ cluster_cfg['cluster']['common']['debugging-reservation-seconds']}}"
 {% if cluster_cfg['rest-server']['github-owner'] %}
         - name: GITHUB_OWNER
           value: {{ cluster_cfg['rest-server']['github-owner'] }}

src/rest-server/src/config/paiConfig.js

@@ -32,12 +32,14 @@ try {
 
 let paiConfigData = {
     machineList: paiMachineList,
+    debuggingReservationSeconds: process.env.DEBUGGING_RESERVATION_SECONDS,
 };
 
 
 // define the schema for pai configuration
 const paiConfigSchema = Joi.object().keys({
     machineList: Joi.array(),
+    debuggingReservationSeconds: Joi.number().integer().positive(),
 }).required();
 
 

src/rest-server/src/models/job.js

@@ -445,6 +445,7 @@ class Job {
           'paiMachineList': paiConfig.machineList,
           'reqAzRDMA': data.jobEnvs && data.jobEnvs.paiAzRDMA === true ? true : false,
           'isDebug': data.jobEnvs && data.jobEnvs.isDebug === true ? true : false,
+          'debuggingReservationSeconds': paiConfig.debuggingReservationSeconds,
         });
     return dockerContainerScript;
   }

src/rest-server/src/templates/dockerContainerScript.mustache

@@ -209,7 +209,7 @@ else
     echo "======          After debugging, please stop the job manually.         ======"
     echo "============================================================================="
 
-    sleep_time=604800
+    sleep_time={{ debuggingReservationSeconds }}
     sleep_count=0
 
     while [ $(( $(date +%s) - $(stat -c %Y /alive/yarn_$PAI_CONTAINER_ID) )) -lt 30 ] && \