[Rest Server] kubernetes pods/nodes list api

src/rest-server/docs/swagger.yaml

@@ -26,6 +26,8 @@ tags:
   description: API endpoint for storage
 - name: job history
   description: API endpoint for job history
+- name: kubernetes
+  description: API endpoint for kubernetes info
 paths:
   /api/docs:
     get:
@@ -1680,6 +1682,54 @@ paths:
           $ref: '#/components/responses/NoJobError'
         501:
           $ref: '#/components/responses/UnknownError'
+  /api/v1/kubernetes/nodes:
+    get:
+      tags:
+      - kubernetes
+      summary: Get kubernetes node list.
+      description: Get kubernetes node list. Need administrator permission.
+      operationId: getK8sNodes
+      security:
+      - bearerAuth: []
+      responses:
+        200:
+          description: Succeeded
+          content:
+            application/json:
+              example:
+              - Please refer to Kubernetes API doc
+              - https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#list-node-v1-core
+        401:
+          $ref: '#/components/responses/UnauthorizedUserError'
+        403:
+          $ref: '#/components/responses/ForbiddenUserError'
+  /api/v1/kubernetes/pods:
+    get:
+      tags:
+      - kubernetes
+      summary: Get kubernetes pod list.
+      description: Get kubernetes pod list. Need administrator permission.
+      operationId: getK8sPods
+      parameters:
+      - name: namespace
+        in: query
+        description: filter pods with namespace
+        schema:
+          type: string
+      security:
+      - bearerAuth: []
+      responses:
+        200:
+          description: Succeeded
+          content:
+            application/json:
+              example:
+              - Please refer to Kubernetes API doc
+              - https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#list-all-namespaces-pod-v1-core
+        401:
+          $ref: '#/components/responses/UnauthorizedUserError'
+        403:
+          $ref: '#/components/responses/ForbiddenUserError'
 
 components:
   parameters:

src/rest-server/package.json

@@ -25,6 +25,7 @@
   },
   "dependencies": {
     "@elastic/elasticsearch": "^7.4.0",
+    "@kubernetes/client-node": "^0.11.0",
     "ajv": "^6.10.0",
     "ajv-merge-patch": "~4.1.0",
     "async": "~2.5.0",
@@ -43,7 +44,6 @@
     "express": "~4.16.2",
     "fs-extra": "~7.0.1",
     "http-errors": "~1.6.3",
-    "http-proxy-middleware": "^0.20.0",
     "joi": "~13.0.1",
     "js-yaml": "^3.13.1",
     "jsonwebtoken": "~8.1.0",

src/rest-server/src/config/kubernetes.js

@@ -18,18 +18,28 @@
 
 const assert = require('assert');
 const {readFileSync} = require('fs');
-const path = require('path');
+const k8s = require('@kubernetes/client-node');
 const logger = require('@pai/config/logger');
 
+const bufferFromFileOrData = (path, data) => {
+  if (path) {
+    return readFileSync(path);
+  } else if (data) {
+    return Buffer.from(data, 'base64');
+  }
+};
+
 const apiserverConfig = {};
 
 const {
   K8S_APISERVER_URI,
   K8S_APISERVER_CA_FILE,
   K8S_APISERVER_TOKEN_FILE,
+  K8S_KUBECONFIG_PATH,
+  RBAC_IN_CLUSTER,
 } = process.env;
 
-if (process.env.RBAC_IN_CLUSTER === 'false') {
+if (RBAC_IN_CLUSTER === 'false') {
   apiserverConfig.uri = K8S_APISERVER_URI;
   // Should be empty
   if (K8S_APISERVER_CA_FILE) {
@@ -41,20 +51,32 @@ if (process.env.RBAC_IN_CLUSTER === 'false') {
     apiserverConfig.token = readFileSync(K8S_APISERVER_TOKEN_FILE, 'utf8');
   }
 } else {
-  const root = process.env.KUBERNETES_CLIENT_SERVICEACCOUNT_ROOT || '/var/run/secrets/kubernetes.io/serviceaccount/';
-  // By default, in rbac enabled k8s, the caPath and tokenPath is a fixed value. However, from the perspective of flexibility,
-  // user can custom the following 2 files' path in the future.
-  const caPath = K8S_APISERVER_CA_FILE || path.join(root, 'ca.crt');
-  const tokenPath = K8S_APISERVER_TOKEN_FILE || path.join(root, 'token');
-  const host = process.env.KUBERNETES_SERVICE_HOST;
-  const port = process.env.KUBERNETES_SERVICE_PORT;
-  apiserverConfig.uri = `https://${host}:${port}`;
-
-  try {
-    // Will be a buffer since SSL context can receive a buffer.
-    apiserverConfig.ca = readFileSync(caPath, 'utf8');
+  if (K8S_APISERVER_CA_FILE) {
+    k8s.Config.SERVICEACCOUNT_CA_PATH = K8S_APISERVER_CA_FILE;
+  }
+  if (K8S_APISERVER_TOKEN_FILE) {
     // Will be a string since http header can only receive a string.
-    apiserverConfig.token = readFileSync(tokenPath, 'utf8');
+    k8s.Config.SERVICEACCOUNT_TOKEN_PATH = K8S_APISERVER_TOKEN_FILE;
+  }
+  try {
+    const kc = new k8s.KubeConfig();
+    if (K8S_KUBECONFIG_PATH) {
+      kc.loadFromFile(K8S_KUBECONFIG_PATH);
+    } else {
+      kc.loadFromDefault();
+    }
+    const cluster = kc.getCurrentCluster();
+    const user = kc.getCurrentUser();
+    apiserverConfig.uri = cluster.server;
+    if (cluster.caFile) {
+      apiserverConfig.ca = readFileSync(cluster.caFile);
+    } else if (cluster.caData) {
+      apiserverConfig.ca = Buffer.from(cluster.caData, 'base64').toString();
+    }
+    apiserverConfig.token = user.token;
+    apiserverConfig.ca = bufferFromFileOrData(cluster.caFile, cluster.caData);
+    apiserverConfig.key = bufferFromFileOrData(user.keyFile, user.keyData);
+    apiserverConfig.cert = bufferFromFileOrData(user.certFile, user.certData);
   } catch (error) {
     logger.error('failed to init rbac config. Please check your clusters\' config');
     throw error;

src/rest-server/src/config/launcher.js

@@ -18,7 +18,6 @@
 
 // module dependencies
 const Joi = require('joi');
-const {apiserver} = require('@pai/config/kubernetes');
 
 
 // define yarn launcher config schema
@@ -90,9 +89,6 @@ const yarnLauncherConfigSchema = Joi.object().keys({
 
 // define k8s launcher config schema
 const k8sLauncherConfigSchema = Joi.object().keys({
-  apiServerUri: Joi.string()
-    .uri()
-    .required(),
   hivedWebserviceUri: Joi.string()
     .uri()
     .required(),
@@ -201,7 +197,6 @@ if (launcherType === 'yarn') {
   launcherConfig.type = launcherType;
 } else if (launcherType === 'k8s') {
   launcherConfig = {
-    apiServerUri: apiserver.uri,
     hivedWebserviceUri: process.env.HIVED_WEBSERVICE_URI,
     apiVersion: 'frameworkcontroller.microsoft.com/v1',
     podGracefulDeletionTimeoutSec: 1800,
@@ -213,31 +208,30 @@ if (launcherType === 'yarn') {
     requestHeaders: {
       'Accept': 'application/json',
       'Content-Type': 'application/json',
-      ...apiserver.token && {Authorization: `Bearer ${apiserver.token}`},
     },
     healthCheckPath: () => {
-      return `${launcherConfig.apiServerUri}/apis/${launcherConfig.apiVersion}`;
+      return `/apis/${launcherConfig.apiVersion}`;
     },
     frameworksPath: (namespace='default') => {
-      return `${launcherConfig.apiServerUri}/apis/${launcherConfig.apiVersion}/namespaces/${namespace}/frameworks`;
+      return `/apis/${launcherConfig.apiVersion}/namespaces/${namespace}/frameworks`;
     },
     frameworkPath: (frameworkName, namespace='default') => {
-      return `${launcherConfig.apiServerUri}/apis/${launcherConfig.apiVersion}/namespaces/${namespace}/frameworks/${frameworkName}`;
+      return `/apis/${launcherConfig.apiVersion}/namespaces/${namespace}/frameworks/${frameworkName}`;
     },
     priorityClassesPath: () => {
-      return `${launcherConfig.apiServerUri}/apis/scheduling.k8s.io/v1/priorityclasses`;
+      return `/apis/scheduling.k8s.io/v1/priorityclasses`;
     },
     priorityClassPath: (priorityClassName) => {
-      return `${launcherConfig.apiServerUri}/apis/scheduling.k8s.io/v1/priorityclasses/${priorityClassName}`;
+      return `/apis/scheduling.k8s.io/v1/priorityclasses/${priorityClassName}`;
     },
     secretsPath: (namespace='default') => {
-      return `${launcherConfig.apiServerUri}/api/v1/namespaces/${namespace}/secrets`;
+      return `/api/v1/namespaces/${namespace}/secrets`;
     },
     secretPath: (secretName, namespace='default') => {
-      return `${launcherConfig.apiServerUri}/api/v1/namespaces/${namespace}/secrets/${secretName}`;
+      return `/api/v1/namespaces/${namespace}/secrets/${secretName}`;
     },
     podPath: (podName, namespace='default') => {
-      return `${launcherConfig.apiServerUri}/api/v1/namespaces/${namespace}/pods/${podName}`;
+      return `/api/v1/namespaces/${namespace}/pods/${podName}`;
     },
   };
 

src/rest-server/src/config/paiConfig.js

@@ -16,14 +16,12 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 // module dependencies
-const axios = require('axios');
-const {Agent} = require('https');
 const Joi = require('joi');
 const yaml = require('js-yaml');
 const {get} = require('lodash');
 const fs = require('fs');
 const logger = require('@pai/config/logger');
-const {apiserver} = require('@pai/config/kubernetes');
+const k8sModel = require('@pai/models/kubernetes');
 
 let paiMachineList = [];
 try {
@@ -57,14 +55,8 @@ paiConfigData = value;
 
 const fetchPAIVersion = async () => {
     try {
-        const res = await axios.request({
-            url: '/api/v1/namespaces/default/configmaps/pai-version',
-            baseURL: apiserver.uri,
-            maxRedirects: 0,
-            httpsAgent: apiserver.ca && new Agent({ca: apiserver.ca}),
-            headers: apiserver.token && {Authorization: `Bearer ${apiserver.token}`},
-        });
-
+        const client = k8sModel.getClient();
+        const res = await client.get('/api/v1/namespaces/default/configmaps/pai-version');
         const version = get(res.data, 'data["PAI.VERSION"]');
         if (version) {
             return version.trim();

src/rest-server/src/config/secret.js

@@ -18,48 +18,25 @@
 
 // module dependencies
 const Joi = require('joi');
-const {readFileSync} = require('fs');
-const {Agent} = require('https');
 const authnConfig = require('@pai/config/authn');
 
 let userSecretConfig = {};
 
 if (authnConfig.authnMethod !== 'OIDC') {
   userSecretConfig = {
-    apiServerUri: process.env.K8S_APISERVER_URI,
     paiUserNameSpace: 'pai-user',
     adminName: process.env.DEFAULT_PAI_ADMIN_USERNAME,
     adminPass: process.env.DEFAULT_PAI_ADMIN_PASSWORD,
   };
 } else {
   userSecretConfig = {
-    apiServerUri: process.env.K8S_APISERVER_URI,
     paiUserNameSpace: 'pai-user',
   };
 }
 
-userSecretConfig.requestConfig = () => {
-  const config = {
-    baseURL: `${userSecretConfig.apiServerUri}/api/v1/namespaces/`,
-    maxRedirects: 0,
-  };
-  if ('K8S_APISERVER_CA_FILE' in process.env) {
-    const ca = readFileSync(process.env.K8S_APISERVER_CA_FILE);
-    config.httpsAgent = new Agent({ca});
-  }
-
-  if ('K8S_APISERVER_TOKEN_FILE' in process.env) {
-    const token = readFileSync(process.env.K8S_APISERVER_TOKEN_FILE, 'ascii');
-    config.headers = {Authorization: `Bearer ${token}`};
-  }
-  return config;
-};
-
 let userSecretConfigSchema = {};
 if (authnConfig.authnMethod !== 'OIDC') {
   userSecretConfigSchema = Joi.object().keys({
-    apiServerUri: Joi.string()
-      .required(),
     paiUserNameSpace: Joi.string()
       .default('pai-user'),
     adminName: Joi.string()
@@ -68,19 +45,11 @@ if (authnConfig.authnMethod !== 'OIDC') {
     adminPass: Joi.string()
       .min(6)
       .required(),
-    requestConfig: Joi.func()
-      .arity(0)
-      .required(),
   }).required();
 } else {
   userSecretConfigSchema = Joi.object().keys({
-    apiServerUri: Joi.string()
-      .required(),
     paiUserNameSpace: Joi.string()
       .default('pai-user'),
-    requestConfig: Joi.func()
-      .arity(0)
-      .required(),
   }).required();
 }
 

src/rest-server/src/middlewares/token.js

@@ -19,6 +19,7 @@ const {userProperty} = require('@pai/config/token');
 const userModel = require('@pai/models/v2/user');
 const tokenModel = require('@pai/models/token');
 const createError = require('@pai/utils/error');
+const logger = require('@pai/config/logger');
 
 const getToken = async (req) => {
   const [scheme, credentials] = req.headers.authorization.split(' ');
@@ -48,6 +49,7 @@ const check = async (req, _, next) => {
     req[userProperty].admin = await userModel.checkAdmin(req[userProperty].username);
     next();
   } catch (error) {
+    logger.debug(error);
     return next(createError('Unauthorized', 'UnauthorizedUserError', 'Your token is invalid.'));
   }
 };