Full vc update (#1985)

* VC update API doc (#1816) * refine api doc * refine doc * refine * refine * [VC update] rest-server (#1831) * change scheduler storage to zookeeper * add queue update * fix * add vc update api * add yarn health check, refine error * add api doc * refine api doc * refine api doc * update yarn response * add unit test * unit test * replace parseint with math.round, fix unit test * replace mathjs with build-in Math * vc update (#1974) * Update yarnContainerScript.mustache add entrypoint="" * add design for vc update * refine design of vc update * add description of /api/v1/:username/virtualClusters * remove design of vc_update to dir of webportal * refine design of vc * VC function development * vc style modification * vc list style modification * Modify the prompt * repair bug * remove design of vc_update and refine annotation * refine var name * Change eslint error * Optimize the code according to pr * Modified read-only
microsoft · Jan 10, 2019 · a24dccb · a24dccb
1 parent 9bfdbb9
commit a24dccb
Show file tree

Hide file tree

Showing 16 changed files with 2,008 additions and 316 deletions.
diff --git a/docs/rest-server/API.md b/docs/rest-server/API.md
@@ -771,10 +771,222 @@ Status: 200
   "numJobs":0,
   "numPendingJobs":0,
   "resourcesUsed":{
-    "memory":0,
-    "vCores":0,
-    "GPUs":0
+   "memory":0,
+   "vCores":0,
+   "GPUs":0
   },
+  "state":"running"
+}
+```
+
+*Response if the virtual cluster does not exist*
+```
+Status: 404
+
+{
+  "code": "NoVirtualClusterError",
+  "message": "Virtual cluster $vcname is not found."
+}
+```
+
+*Response if a server error occured*
+```
+Status: 500
+
+{
+  "code": "UnknownError",
+  "message": "*Upstream error messages*"
+}
+```
+
+### `PUT virtual-clusters/:vcName` (administrator only)
+
+Add or update virtual cluster quota in the system, don't allow to operate "default" vc.
+
+*Request*
+```
+PUT /api/v1/virtual-clusters/:vcName
+Authorization: Bearer <ACCESS_TOKEN>
+```
+
+*Parameters*
+```
+{
+  "vcCapacity": new capacity
+}
+```
+
+*Response if succeeded*
+```
+Status: 201
+
+{
+  "message": "Update vc: $vcName to capacity: $vcCapacity successfully."
+}
+```
+
+*Response if try to update "default" vc*
+```
+Status: 403
+
+{
+  "code": "ForbiddenUserError",
+  "message": "Don't allow to update default vc"
+}
+```
+
+*Response if current user has no permission*
+```
+Status: 403
+
+{
+  "code": "ForbiddenUserError",
+  "message": "Non-admin is not allow to do this operation."
+}
+```
+
+*Response if no enough quota*
+```
+Status: 403
+
+{
+  "code": "NoEnoughQuotaError",
+  "message": "No enough quota in default vc."
+}
+```
+
+*Response if "default" virtual cluster does not exist*
+```
+Status: 404
+
+{
+  "code": "NoVirtualClusterError",
+  "message": "Default virtual cluster is not found, can't allocate or free resource."
+}
+```
+
+*Response if a server error occured*
+```
+Status: 500
+
+{
+  "code": "UnknownError",
+  "message": "*Upstream error messages*"
+}
+```
+
+### `DELETE virtual-clusters/:vcName` (administrator only)
+
+remove virtual cluster in the system, don't allow to operate "default" vc.
+
+*Request*
+```
+DELETE /api/v1/virtual-clusters/:vcName
+Authorization: Bearer <ACCESS_TOKEN>
+```
+
+*Response if succeeded*
+```
+Status: 201
+
+{
+  "message": "Remove vc: $vcName successfully."
+}
+```
+
+*Response if current user has no permission*
+```
+Status: 403
+
+{
+  "code": "ForbiddenUserError",
+  "message": "Non-admin is not allow to do this operation."
+}
+```
+
+*Response if try to update "default" vc*
+```
+Status: 403
+
+{
+  "code": "ForbiddenUserError",
+  "message": "Don't allow to remove default vc."
+}
+```
+
+*Response if the virtual cluster does not exist*
+```
+Status: 404
+
+{
+  "code": "NoVirtualClusterError",
+  "message": "Virtual cluster $vcname is not found."
+}
+```
+
+*Response if "default" virtual cluster does not exist*
+```
+Status: 404
+
+{
+  "code": "NoVirtualClusterError",
+  "message": "Default virtual cluster is not found, can't allocate or free resource."
+}
+```
+
+*Response if a server error occured*
+```
+Status: 500
+
+{
+  "code": "UnknownError",
+  "message": "*Upstream error messages*"
+}
+```
+
+### `PUT virtual-clusters/:vcName/status` (administrator only)
+
+Change virtual cluster status, don't allow to operate "default" vc.
+
+*Request*
+```
+PUT /api/v1/virtual-clusters/:vcName/status
+Authorization: Bearer <ACCESS_TOKEN>
+```
+
+*Parameters*
+```
+{
+  "vcStatus": "running" | "stopped"
+}
+```
+
+*Response if succeeded*
+```
+Status: 201
+
+{
+  "message": "Update vc: $vcName to status: $vcStatus successfully."
+}
+```
+
+*Response if try to update "default" vc*
+```
+Status: 403
+
+{
+  "code": "ForbiddenUserError",
+  "message": "Don't allow to update default vc"
+}
+```
+
+*Response if current user has no permission*
+```
+Status: 403
+
+{
+  "code": "ForbiddenUserError",
+  "message": "Non-admin is not allow to do this operation."
 }
 ```
 
@@ -806,4 +1018,4 @@ the version upgrade, which has no namespaces. They are called "legacy jobs", whi
 but cannot be created. To figure out them, there is a "legacy: true" field of them in list apis.
 
 In the next versions, all operations of legacy jobs may be disabled, so please re-create them as namespaced
-job as soon as possible.
+job as soon as possible.
diff --git a/src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml b/src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/yarn-site.xml
@@ -44,6 +44,12 @@
         <value>1048576</value>
         <description>default is 8GB, here we set 1024G</description>
     </property>
+
+    <property>
+        <name>yarn.scheduler.configuration.store.class</name>
+        <value>zk</value>
+        <description>default is file, change it to zk to enable config by rest api</description>
+    </property>
 
     <property>
       <name>yarn.resourcemanager.scheduler.class</name>

diff --git a/src/rest-server/package.json b/src/rest-server/package.json
@@ -50,10 +50,11 @@
     "node-cache": "~4.2.0",
     "node-etcd": "~5.1.0",
     "nyc": "~11.6.0",
+    "ssh-keygen": "~0.4.2",
     "statuses": "~1.5.0",
     "unirest": "~0.5.1",
     "winston": "~2.4.0",
-    "ssh-keygen": "~0.4.2"
+    "xml2js": "~0.4.19"
   },
   "scripts": {
     "coveralls": "nyc report --reporter=text-lcov | coveralls ..",

diff --git a/src/rest-server/src/config/vc.js b/src/rest-server/src/config/vc.js
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation
+// All rights reserved.
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+// documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+// to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// module dependencies
+const Joi = require('joi');
+
+// define the input schema for the 'update vc' api
+const vcPutInputSchema = Joi.object().keys({
+  vcCapacity: Joi.number()
+    .integer()
+    .min(0)
+    .max(100)
+    .required(),
+}).required();
+
+// define the input schema for the 'put vc status' api
+const vcStatusPutInputSchema = Joi.object().keys({
+  vcStatus: Joi.string()
+    .valid(['stopped', 'running'])
+    .required(),
+}).required();
+
+// module exports
+module.exports = {
+  vcPutInputSchema: vcPutInputSchema,
+  vcStatusPutInputSchema: vcStatusPutInputSchema,
+};
diff --git a/src/rest-server/src/config/yarn.js b/src/rest-server/src/config/yarn.js
@@ -18,6 +18,9 @@
 
 // module dependencies
 const Joi = require('joi');
+const unirest = require('unirest');
+const config = require('./index');
+const logger = require('./logger');
 
 // get config from environment variables
 let yarnConfig = {
@@ -26,8 +29,13 @@ let yarnConfig = {
     'Accept': 'application/json',
   },
   yarnVcInfoPath: `${process.env.YARN_URI}/ws/v1/cluster/scheduler`,
+  webserviceUpdateQueueHeaders: {
+    'Content-Type': 'application/xml',
+  },
+  yarnVcUpdatePath: `${process.env.YARN_URI}/ws/v1/cluster/scheduler-conf`,
 };
 
+
 const yarnConfigSchema = Joi.object().keys({
   yarnUri: Joi.string()
     .uri()
@@ -37,6 +45,11 @@ const yarnConfigSchema = Joi.object().keys({
   yarnVcInfoPath: Joi.string()
     .uri()
     .required(),
+  webserviceUpdateQueueHeaders: Joi.object()
+    .required(),
+  yarnVcUpdatePath: Joi.string()
+    .uri()
+    .required(),
 }).required();
 
 const {error, value} = Joi.validate(yarnConfig, yarnConfigSchema);
@@ -45,4 +58,18 @@ if (error) {
 }
 yarnConfig = value;
 
+
+// framework launcher health check
+if (config.env !== 'test') {
+  unirest.get(yarnConfig.yarnVcInfoPath)
+  .timeout(2000)
+  .end((res) => {
+    if (res.status === 200) {
+      logger.info('connected to yarn successfully');
+    } else {
+      throw new Error('cannot connect to yarn');
+    }
+  });
+}
+
 module.exports = yarnConfig;