From ecc4aaa9a1a9f67da366926094714f827307469e Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Mon, 20 Jan 2020 17:29:35 +0800 Subject: [PATCH 01/12] Refactor storage by leveraging persistent volumes Refactor storage by leveraging kubernetes pv and pvc. --- src/rest-server/docs/swagger.yaml | 362 +++--------------- src/rest-server/src/controllers/v2/storage.js | 213 +---------- src/rest-server/src/controllers/v2/user.js | 11 - src/rest-server/src/middlewares/cache.js | 66 ---- src/rest-server/src/models/v2/group.js | 8 +- src/rest-server/src/models/v2/job/k8s.js | 50 ++- src/rest-server/src/models/v2/storage.js | 162 ++++++-- src/rest-server/src/models/v2/user.js | 20 +- src/rest-server/src/routes/v2/index.js | 3 +- src/rest-server/src/routes/v2/storage.js | 55 +-- src/rest-server/src/utils/dbUtil.js | 32 -- src/rest-server/src/utils/error.d.ts | 1 + src/rest-server/src/utils/localCache.js | 87 ----- src/rest-server/src/utils/storageBase.js | 31 -- src/rest-server/src/utils/userSecret.js | 132 ------- 15 files changed, 258 insertions(+), 975 deletions(-) delete mode 100644 src/rest-server/src/middlewares/cache.js delete mode 100644 src/rest-server/src/utils/dbUtil.js delete mode 100644 src/rest-server/src/utils/localCache.js delete mode 100644 src/rest-server/src/utils/storageBase.js delete mode 100644 src/rest-server/src/utils/userSecret.js diff --git a/src/rest-server/docs/swagger.yaml b/src/rest-server/docs/swagger.yaml index 4bcfd44b86..361d65e8cf 100644 --- a/src/rest-server/docs/swagger.yaml +++ b/src/rest-server/docs/swagger.yaml @@ -1317,165 +1317,31 @@ paths: $ref: '#/components/responses/NoJobConfigError/content/application~1json/examples/NoJobConfigError' 500: $ref: '#/components/responses/UnknownError' - /api/v2/storage/server/{storage}: + /api/v2/storages: get: tags: - storage - summary: Get storage server data in the system. - description: Get storage server data in the system. - operationId: getStorageServer + summary: Get storage list (persistent volume claims) for current user. + description: Get storage list for which current user has permissions. + operationId: getStorages security: - bearerAuth: [] - parameters: - - $ref: '#/components/parameters/storage' - responses: - 200: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Storage' - 500: - $ref: '#/components/responses/UnknownError' - delete: - tags: - - storage - summary: Remove storage server in the system. - description: Remove storage server in the system. Storage server empty is system reserved and cannot be removed. - operationId: removeStorageServer - security: - - bearerAuth: [] - parameters: - - $ref: '#/components/parameters/storage' - responses: - 200: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - example: - message: Storage Server is deleted successfully - 403: - description: ForbiddenUserError or ForbiddenKeyError - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - examples: - ForbiddenUserError: - $ref: '#/components/responses/ForbiddenUserError/content/application~1json/examples/ForbiddenUserError' - ForbiddenKeyError: - $ref: '#/components/responses/ForbiddenKeyError/content/application~1json/examples/ForbiddenKeyError' - 500: - $ref: '#/components/responses/UnknownError' - /api/v2/storage/server: - get: - tags: - - storage - summary: Get storage server data in the system. - description: Given storage server names, find server data. - operationId: getStorageServers - security: - - bearerAuth: [] - parameters: - - name: names - in: query - description: filter storage server with names, default name empty will be ignored - schema: - type: string responses: 200: description: Succeeded content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/Storage' - 500: - $ref: '#/components/responses/UnknownError' - post: - tags: - - storage - summary: Create storage server in system. - description: Create storage server in system. - operationId: createStorageServer - security: - - bearerAuth: [] - requestBody: - description: Storage server - content: - application/json: - schema: - $ref: '#/components/schemas/Storage' - required: true - responses: - 201: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - example: - message: Storage Server is created successfully - 403: - description: ForbiddenUserError or ForbiddenKeyError - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - examples: - ForbiddenUserError: - $ref: '#/components/responses/ForbiddenUserError/content/application~1json/examples/ForbiddenUserError' - ForbiddenKeyError: - $ref: '#/components/responses/ForbiddenKeyError/content/application~1json/examples/ForbiddenKeyError' - 500: - $ref: '#/components/responses/UnknownError' - put: - tags: - - storage - summary: Update storage server in system. - description: Update storage server in system. Storage server empty is system reserved and cannot be updated. - operationId: updateStorageServer - security: - - bearerAuth: [] - requestBody: - description: Storage server - content: - application/json: - schema: - $ref: '#/components/schemas/Storage' - required: true - responses: - 201: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - example: - message: Storage Server is updated successfully - 403: - description: ForbiddenUserError or ForbiddenKeyError - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - examples: - ForbiddenUserError: - $ref: '#/components/responses/ForbiddenUserError/content/application~1json/examples/ForbiddenUserError' - ForbiddenKeyError: - $ref: '#/components/responses/ForbiddenKeyError/content/application~1json/examples/ForbiddenKeyError' + $ref: '#/components/schemas/StorageSummary' 500: $ref: '#/components/responses/UnknownError' - /api/v2/storage/config/{storage}: + /api/v2/storages/{storage}: get: tags: - storage - summary: Get storage config data in the system. - description: Get storage config data in the system. - operationId: getStorageConfig + summary: Get storage (persistent volume claim) for the given name. + description: Get storage for the given name. + operationId: getStorage security: - bearerAuth: [] parameters: @@ -1486,139 +1352,20 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/StorageConfig' - 500: - $ref: '#/components/responses/UnknownError' - delete: - tags: - - storage - summary: Remove storage config in the system. - description: Remove storage config in the system. Storage config empty is system reserved and cannot be removed. - operationId: removeStorageConfig - security: - - bearerAuth: [] - parameters: - - $ref: '#/components/parameters/storage' - responses: - 201: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - example: - message: Storage Config is deleted successfully - 403: - description: ForbiddenUserError or ForbiddenKeyError - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - examples: - ForbiddenUserError: - $ref: '#/components/responses/ForbiddenUserError/content/application~1json/examples/ForbiddenUserError' - ForbiddenKeyError: - $ref: '#/components/responses/ForbiddenKeyError/content/application~1json/examples/ForbiddenKeyError' - 500: - $ref: '#/components/responses/UnknownError' - /api/v2/storage/config: - get: - tags: - - storage - summary: Get storage config data in the system. - description: Given storage config names, find server data. - operationId: getStorageConfigs - security: - - bearerAuth: [] - parameters: - - name: names - in: query - description: filter storage server with names, default name empty will be ignored - schema: - type: string - responses: - 200: - description: Succeeded - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/StorageConfig' - 500: - $ref: '#/components/responses/UnknownError' - post: - tags: - - storage - summary: Create storage config in system. - description: Create storage config in system. - operationId: createStorageConfig - security: - - bearerAuth: [] - requestBody: - description: Storage config - content: - application/json: - schema: - $ref: '#/components/schemas/StorageConfig' - required: true - responses: - 201: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - example: - message: Storage Config is created successfully + $ref: '#/components/schemas/StorageDetail' 403: - description: ForbiddenUserError or ForbiddenKeyError - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - examples: - ForbiddenUserError: - $ref: '#/components/responses/ForbiddenUserError/content/application~1json/examples/ForbiddenUserError' - ForbiddenKeyError: - $ref: '#/components/responses/ForbiddenKeyError/content/application~1json/examples/ForbiddenKeyError' - 500: - $ref: '#/components/responses/UnknownError' - put: - tags: - - storage - summary: Update storage config in system. - description: Update storage config in system. Storage config empty is system reserved and cannot be updated. - operationId: updateStorageConfig - security: - - bearerAuth: [] - requestBody: - description: Storage config - content: - application/json: - schema: - $ref: '#/components/schemas/Storage' - required: true - responses: - 201: - description: Succeeded - content: - application/json: - schema: - $ref: '#/components/schemas/Response' - example: - message: Storage Config is updated successfully - 403: - description: ForbiddenUserError or ForbiddenKeyError + description: ForbiddenUserError content: application/json: schema: $ref: '#/components/schemas/Response' examples: ForbiddenUserError: - $ref: '#/components/responses/ForbiddenUserError/content/application~1json/examples/ForbiddenUserError' - ForbiddenKeyError: - $ref: '#/components/responses/ForbiddenKeyError/content/application~1json/examples/ForbiddenKeyError' + value: + code: ForbiddenUserError + message: User {user} is not allowed to access {storage}. + 404: + $ref: '#/components/responses/NoStorageError' 500: $ref: '#/components/responses/UnknownError' /api/v2/jobs/{user}~{job}/job-attempts/healthz: @@ -1871,52 +1618,48 @@ components: type: string enum: ['RUNNING', 'STOPPED', 'DRAINING'] description: RUNNING -> vc is enabled, STOPPED -> vc is disabled, without either new job or running job, DRAINING -> intermedia state from RUNNING to STOPPED, in waiting on existing job. - Storage: + StorageSummary: type: object properties: - spn: + storages: + type: array + items: + type: object + properties: + name: + type: string + share: + type: boolean + volumeName: + type: string + required: [name, share, volumeName] + required: [storages] + StorageDetail: + type: object + properties: + name: + type: string + share: + type: boolean + volumeName: type: string type: type: string + enum: + - nfs + - samba + - azureFile + - azureBlob + - unknown data: type: object - properties: - spn: - type: string - type: - type: string - address: - type: string - rootPath: - type: string - extension: - type: object - required: [spn, type, data, extension] - StorageConfig: - type: object - properties: - name: + secretName: type: string - default: - type: boolean - servers: + mountOptions: type: array items: type: string - mountInfos: - type: array - items: - type: object - properties: - mountPoint: - type: string - path: - type: string - server: - type: string - permission: - type: string - required: [name, default, servers, mountInfos] + required: [name, share, volumeName] JobAttempt: type: object description: TODO @@ -1943,6 +1686,17 @@ components: value: code: NoVirtualClusterError message: Virtual cluster {vc} is not found. + NoStorageError: + description: NoStorageError + content: + application/json: + schema: + $ref: '#/components/schemas/Response' + examples: + NoVirtualClusterError: + value: + code: NoStorageError + message: Storage {storage} is not found. UnauthorizedUserError: description: UnauthorizedUserError content: diff --git a/src/rest-server/src/controllers/v2/storage.js b/src/rest-server/src/controllers/v2/storage.js index 11be225c12..12640c3927 100644 --- a/src/rest-server/src/controllers/v2/storage.js +++ b/src/rest-server/src/controllers/v2/storage.js @@ -16,206 +16,27 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // module dependencies -const createError = require('@pai/utils/error'); -const storageModel = require('@pai/models/v2/storage'); -const {isArray, isEmpty} = require('lodash'); +const asyncHandler = require('@pai/middlewares/v2/asyncHandler'); +const storage = require('@pai/models/v2/storage'); -const getStorageServer = async (req, res, next) => { - try { - const name = req.params.name; - const storageServerInfo = await storageModel.getStorageServer(name); - return res.status(200).json(storageServerInfo); - } catch (error) { - return next(createError.unknown(error)); - } -}; -const getStorageServers = async (req, res, next) => { - try { - const names = isEmpty(req.query.names) - ? [] - : isArray(req.query.names) - ? req.query.names - : [req.query.names]; - const storageServerList = await storageModel.getStorageServers(names); - return res.status(200).json(storageServerList); - } catch (error) { - return next(createError.unknown(error)); - } -}; +const list = asyncHandler(async (req, res) => { + const userName = req.user.username; + const admin = req.user.admin; + const data = await storage.list(admin ? undefined : userName); + res.json(data); +}); -const getStorageConfig = async (req, res, next) => { - try { - const name = req.params.name; - const storageConfigInfo = await storageModel.getStorageConfig(name); - return res.status(200).json(storageConfigInfo); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const getStorageConfigs = async (req, res, next) => { - try { - const names = isEmpty(req.query.names) - ? [] - : isArray(req.query.names) - ? req.query.names - : [req.query.names]; - const storageConfigList = await storageModel.getStorageConfigs(names); - return res.status(200).json(storageConfigList); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const createStorageServer = async (req, res, next) => { - try { - if (!req.user.admin) { - next( - createError( - 'Forbidden', - 'ForbiddenUserError', - `Non-admin is not allow to do this operation.` - ) - ); - } - const name = req.body.spn; - const value = { - spn: req.body.spn, - type: req.body.type, - data: req.body.data, - }; - await storageModel.createStorageServer(name, value); - return res.status(201).json({ - message: 'Storage Server is created successfully', - }); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const updateStorageServer = async (req, res, next) => { - try { - if (!req.user.admin) { - next( - createError( - 'Forbidden', - 'ForbiddenUserError', - `Non-admin is not allow to do this operation.` - ) - ); - } - const name = req.body.spn; - const value = { - spn: req.body.spn, - type: req.body.type, - data: req.body.data, - }; - await storageModel.updateStorageServer(name, value); - return res.status(201).json({ - message: 'Storage Server is updated successfully', - }); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const deleteStorageServer = async (req, res, next) => { - try { - if (!req.user.admin) { - next( - createError( - 'Forbidden', - 'ForbiddenUserError', - `Non-admin is not allow to do this operation.` - ) - ); - } - const name = req.params.name; - await storageModel.deleteStorageServer(name); - return res.status(201).json({ - message: 'Storage Server is deleted successfully', - }); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const createStorageConfig = async (req, res, next) => { - try { - if (!req.user.admin) { - next( - createError( - 'Forbidden', - 'ForbiddenUserError', - `Non-admin is not allow to do this operation.` - ) - ); - } - const name = req.body.name; - const value = req.body; - await storageModel.createStorageConfig(name, value); - return res.status(201).json({ - message: 'Storage Config is created successfully', - }); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const updateStorageConfig = async (req, res, next) => { - try { - if (!req.user.admin) { - next( - createError( - 'Forbidden', - 'ForbiddenUserError', - `Non-admin is not allow to do this operation.` - ) - ); - } - const name = req.body.name; - const value = req.body; - await storageModel.updateStorageConfig(name, value); - return res.status(201).json({ - message: 'Storage Config is updated successfully', - }); - } catch (error) { - return next(createError.unknown(error)); - } -}; - -const deleteStorageConfig = async (req, res, next) => { - try { - if (!req.user.admin) { - next( - createError( - 'Forbidden', - 'ForbiddenUserError', - `Non-admin is not allow to do this operation.` - ) - ); - } - const name = req.params.name; - await storageModel.deleteStorageConfig(name); - return res.status(201).json({ - message: 'Storage Config is deleted successfully', - }); - } catch (error) { - return next(createError.unknown(error)); - } -}; +const get = asyncHandler(async (req, res) => { + const storageName = req.params.storageName + const userName = req.user.username; + const admin = req.user.admin; + const data = await storage.get(storageName, admin ? undefined : userName); + res.json(data); +}); // module exports module.exports = { - getStorageServer, - getStorageServers, - getStorageConfig, - getStorageConfigs, - createStorageServer, - updateStorageServer, - deleteStorageServer, - createStorageConfig, - updateStorageConfig, - deleteStorageConfig, + list, + get, }; diff --git a/src/rest-server/src/controllers/v2/user.js b/src/rest-server/src/controllers/v2/user.js index 03c2379bf8..83546a0ff4 100644 --- a/src/rest-server/src/controllers/v2/user.js +++ b/src/rest-server/src/controllers/v2/user.js @@ -35,16 +35,6 @@ const getUserVCs = async (username) => { return [...virtualClusters]; }; -const getUserStorageConfigs = async (username) => { - const userInfo = await userModel.getUser(username); - let storageConfigs = new Set(); - for (const group of userInfo.grouplist) { - const groupStorageConfigs = await groupModel.getGroupStorageConfigs(group); - storageConfigs = new Set([...storageConfigs, ...groupStorageConfigs]); - } - return [...storageConfigs]; -}; - const getUser = async (req, res, next) => { try { const username = req.params.username; @@ -472,5 +462,4 @@ module.exports = { updateUserPassword, createUser, getUserVCs, - getUserStorageConfigs, }; diff --git a/src/rest-server/src/middlewares/cache.js b/src/rest-server/src/middlewares/cache.js deleted file mode 100644 index adca4ca839..0000000000 --- a/src/rest-server/src/middlewares/cache.js +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -const dbUtility = require('@pai/utils/dbUtil'); -const logger = require('@pai/config/logger'); - -/** - * A K-V store with 10-min timeout. - * Key: HTTP requested path. - * Val: { code: xxx, data: yyy } - */ -const cache = dbUtility.getStorageObject('localCache', { - ttlSeconds: 600, -}); - -const wrapWithCache = (handler) => { - return function(req, res) { - let key = req.originalUrl; - cache.get(key, null, function(err1, val1) { - if (err1 || !val1) { - handler(req, function(err2, val2) { - if (err2) { - // Double check because other request may fill in cache already - cache.get(key, null, function(err3, val3) { - if (err3 || !val3) { - logger.error(err3); - res.status(err2.code).json({ - message: err2.message, - }); - } else { - res.status(val3.code).json(val3.data); - } - }); - } else { - cache.set(key, val2, null, function(err3, _) { - if (err3) { - logger.error(err3); - } - res.status(val2.code).json(val2.data); - }); - } - }); - } else { - logger.debug(`hit cache with path "${key}"`); - res.status(val1.code).json(val1.data); - } - }); - }; -}; - -module.exports = wrapWithCache; diff --git a/src/rest-server/src/models/v2/group.js b/src/rest-server/src/models/v2/group.js index 8dc8f0b424..ee51011790 100644 --- a/src/rest-server/src/models/v2/group.js +++ b/src/rest-server/src/models/v2/group.js @@ -112,12 +112,12 @@ const getStorageConfigsWithGroupInfo = async (groupItems) => { return [...storageConfigs]; }; -const getGroupStorageConfigs = async (groupname) => { +const getGroupStorages = async (groupname) => { const groupItem = await getGroup(groupname); return getStorageConfigsWithGroupInfo([groupItem]); }; -const getGroupsStorageConfigs = async (grouplist) => { +const getGroupsStorages = async (grouplist) => { const groupItems = await getListGroup(grouplist); return getStorageConfigsWithGroupInfo(groupItems); }; @@ -489,8 +489,8 @@ module.exports = { getGroupVCs, getGroupsVCs, getVCsWithGroupInfo, - getGroupStorageConfigs, - getGroupsStorageConfigs, + getGroupStorages, + getGroupsStorages, getStorageConfigsWithGroupInfo, filterExistGroups, }; diff --git a/src/rest-server/src/models/v2/job/k8s.js b/src/rest-server/src/models/v2/job/k8s.js index 08056de41e..c996cfd2b6 100644 --- a/src/rest-server/src/models/v2/job/k8s.js +++ b/src/rest-server/src/models/v2/job/k8s.js @@ -28,6 +28,7 @@ const launcherConfig = require('@pai/config/launcher'); const createError = require('@pai/utils/error'); const protocolSecret = require('@pai/utils/protocolSecret'); const userModel = require('@pai/models/v2/user'); +const storageModel = require('@pai/models/v2/storage'); const k8sModel = require('@pai/models/kubernetes/kubernetes'); const k8sSecret = require('@pai/models/kubernetes/k8s-secret'); const env = require('@pai/utils/env'); @@ -338,7 +339,7 @@ const convertFrameworkDetail = async (framework) => { return detail; }; -const generateTaskRole = (frameworkName, taskRole, jobInfo, frameworkEnvList, config, storageConfig) => { +const generateTaskRole = (frameworkName, taskRole, jobInfo, frameworkEnvList, config) => { const ports = config.taskRoles[taskRole].resourcePerInstance.ports || {}; for (let port of ['ssh', 'http']) { if (!(port in ports)) { @@ -428,10 +429,6 @@ const generateTaskRole = (frameworkName, taskRole, jobInfo, frameworkEnvList, co name: 'GANG_ALLOCATION', value: gangAllocation, }, - { - name: 'STORAGE_CONFIGS', - value: JSON.stringify(storageConfig), - }, ...frameworkEnvList, ...taskRoleEnvList, ], @@ -573,6 +570,25 @@ const generateTaskRole = (frameworkName, taskRole, jobInfo, frameworkEnvList, co name: `${encodeName(frameworkName)}-regcred`, }); } + // add storages + if ('extras' in config && config.extras.storages) { + for (let storage of config.extras.storages) { + if (!storage.name) { + continue; + } + frameworkTaskRole.task.pod.spec.containers[0].volumeMounts.push({ + name: `${storage.name}-volume`, + mountPath: storage.mountPath || `/mnt/${storage.name}`, + ...storage.share === false && {subPath: `${jobInfo.userName}`}, + }); + frameworkTaskRole.task.pod.spec.volumes.push({ + name: `${storage.name}-volume`, + persistentVolumeClaim: { + claimName: `${storage.name}`, + }, + }); + } + } // fill in completion policy const completion = config.taskRoles[taskRole].completion; frameworkTaskRole.frameworkAttemptCompletionPolicy = { @@ -614,7 +630,7 @@ const generateTaskRole = (frameworkName, taskRole, jobInfo, frameworkEnvList, co return frameworkTaskRole; }; -const generateFrameworkDescription = (frameworkName, virtualCluster, config, rawConfig, storageConfig) => { +const generateFrameworkDescription = (frameworkName, virtualCluster, config, rawConfig) => { const [userName, jobName] = frameworkName.split(/~(.+)/); const jobInfo = { jobName, @@ -655,7 +671,7 @@ const generateFrameworkDescription = (frameworkName, virtualCluster, config, raw let totalGpuNumber = 0; for (let taskRole of Object.keys(config.taskRoles)) { totalGpuNumber += config.taskRoles[taskRole].resourcePerInstance.gpu * config.taskRoles[taskRole].instances; - const taskRoleDescription = generateTaskRole(frameworkName, taskRole, jobInfo, frameworkEnvList, config, storageConfig); + const taskRoleDescription = generateTaskRole(frameworkName, taskRole, jobInfo, frameworkEnvList, config); if (launcherConfig.enabledPriorityClass) { taskRoleDescription.task.pod.spec.priorityClassName = `${encodeName(frameworkName)}-priority`; } @@ -877,8 +893,24 @@ const put = async (frameworkName, config, rawConfig) => { throw createError('Forbidden', 'ForbiddenUserError', `User ${userName} is not allowed to do operation in ${virtualCluster}`); } - const storageConfig = await userModel.getUserStorageConfigs(userName); - const frameworkDescription = generateFrameworkDescription(frameworkName, virtualCluster, config, rawConfig, storageConfig); + // check storages for current user + if ('extras' in config && config.extras.storages) { + const userStorages = {}; + (await storageModel.list()).storages + .forEach(userStorage => userStorages[userStorage.name] = userStorage); + for (let storage of config.extras.storages) { + if (!storage.name) { + continue; + } + if (!(storage.name in userStorages)) { + throw createError('Not Found', 'NoStorageError', `Storage ${storage.name} is not found.`); + } else { + storage.share = userStorages[storage.name].share; + } + } + } + + const frameworkDescription = generateFrameworkDescription(frameworkName, virtualCluster, config, rawConfig); // generate image pull secret const auths = Object.values(config.prerequisites.dockerimage) diff --git a/src/rest-server/src/models/v2/storage.js b/src/rest-server/src/models/v2/storage.js index 36a7af0e97..c5363ba85f 100644 --- a/src/rest-server/src/models/v2/storage.js +++ b/src/rest-server/src/models/v2/storage.js @@ -16,62 +16,140 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // module dependencies -const crudUtil = require('@pai/utils/manager/storage/crudUtil'); +const status = require('statuses'); +const createError = require('@pai/utils/error'); +const user = require('@pai/models/v2/user'); +const kubernetes = require('@pai/models/kubernetes/kubernetes'); -const crudType = 'k8sSecret'; -const crudStorage = crudUtil.getStorageObject(crudType); -// crud storage wrappers -const getStorageServer = async (name) => { - return await crudStorage.readStorageServer(name); +const convertVolumeSummary = (pvc) => { + return { + name: pvc.metadata.name, + share: (pvc.metadata.labels && pvc.metadata.labels.share === 'false') ? false : true, + volumeName: pvc.spec.volumeName, + }; }; -const getStorageServers = async (names) => { - return await crudStorage.readStorageServers(names); -}; - -const getStorageConfig = async (name) => { - return await crudStorage.readStorageConfig(name); -}; - -const getStorageConfigs = async (names) => { - return await crudStorage.readStorageConfigs(names); -}; +const convertVolumeDetail = async (pvc) => { + const storage = convertVolumeSummary(pvc); + if (!storage.volumeName) { + return storage; + } -const createStorageServer = async (name, value) => { - return await crudStorage.createStorageServer(name, value); -}; + let response; + try { + response = await kubernetes.getClient().get( + `/api/v1/persistentvolumes/${storage.volumeName}`, + ); + } catch (error) { + if (error.response != null) { + response = error.response; + } else { + throw error; + } + } + if (response.status !== status('OK')) { + throw createError(response.status, 'UnknownError', response.data.message); + } -const createStorageConfig = async (name, value) => { - return await crudStorage.createStorageConfig(name, value); + const pv = response.data; + if (pv.spec.nfs) { + storage.type = 'nfs'; + storage.data = { + server: pv.spec.nfs.server, + path: pv.spec.nfs.path, + }; + storage.mountOptions = pv.spec.mountOptions; + } else if (pv.spec.azureFile) { + storage.type = 'azureFile'; + storage.data = { + shareName: pv.spec.shareName, + }; + storage.secretName = pv.spec.secretName; + } else if (pv.spec.flexVolume) { + if (pv.spec.flexVolume.driver === 'azure/blobfuse') { + storage.type = 'azureBlob'; + storage.data = { + containerName: pv.spec.flexVolume.options.container, + }; + } else if (pv.spec.flexVolume.driver === 'microsoft.com/smb') { + storage.type = 'samba'; + storage.data = { + address: pv.spec.flexVolume.options.source, + }; + } else { + storage.type = 'other'; + storage.data = {}; + } + if (pv.spec.flexVolume.secretRef) { + storage.secretName = pv.spec.flexVolume.secretRef.name; + } + if (pv.spec.flexVolume.options.mountoptions) { + storage.mountOptions = pv.spec.flexVolume.options.mountoptions.split(','); + } + } else { + storage.type = 'unknown'; + storage.data = {}; + } + return storage; }; -const updateStorageServer = async (name, value) => { - return await crudStorage.updateStorageServer(name, value); -}; +const list = async (userName) => { + let response; + try { + response = await kubernetes.getClient().get( + '/api/v1/namespaces/default/persistentvolumeclaims', + ); + } catch (error) { + if (error.response != null) { + response = error.response; + } else { + throw error; + } + } + if (response.status !== status('OK')) { + throw createError(response.status, 'UnknownError', response.data.message); + } -const updateStorageConfig = async (name, value) => { - return await crudStorage.updateStorageConfig(name, value); + const userStorages = userName ? await user.getUserStorages(userName) : undefined; + const storages = response.data.items + .filter((item) => item.status.phase === 'Bound') + .filter((item) => userStorages === undefined || userStorages.includes(item.metadata.name)) + .map(convertVolumeSummary); + return {storages}; }; -const deleteStorageServer = async (name) => { - return await crudStorage.removeStorageServer(name); -}; +const get = async (storageName, userName) => { + let response; + try { + response = await kubernetes.getClient().get( + `/api/v1/namespaces/default/persistentvolumeclaims/${storageName}`, + ); + } catch (error) { + if (error.response != null) { + response = error.response; + } else { + throw error; + } + } -const deleteStorageConfig = async (name) => { - return await crudStorage.removeStorageConfig(name); + if (response.status === status('OK')) { + const pvc = response.data; + if (!userName || (await user.checkUserStorage(userName, pvc.metadata.name))) { + return convertVolumeDetail(pvc); + } else { + throw createError('Forbidden', 'ForbiddenUserError', `User ${userName} is not allowed to access ${storageName}.`); + } + } + if (response.status === status('Not Found')) { + throw createError('Not Found', 'NoStorageError', `Storage ${storageName} is not found.`); + } else { + throw createError(response.status, 'UnknownError', response.data.message); + } }; // module exports module.exports = { - getStorageServer, - getStorageServers, - getStorageConfig, - getStorageConfigs, - createStorageServer, - createStorageConfig, - updateStorageServer, - updateStorageConfig, - deleteStorageServer, - deleteStorageConfig, + list, + get, }; diff --git a/src/rest-server/src/models/v2/user.js b/src/rest-server/src/models/v2/user.js index 4855619d66..489c454c5f 100644 --- a/src/rest-server/src/models/v2/user.js +++ b/src/rest-server/src/models/v2/user.js @@ -74,11 +74,6 @@ const getUserVCs = async (username) => { return groupModel.getGroupsVCs(userItem.grouplist); }; -const getUserStorageConfigs = async (username) => { - const userItem = await getUser(username); - return groupModel.getGroupsStorageConfigs(userItem.grouplist); -}; - const checkAdmin = async (username) => { const userItem = await getUser(username); return groupModel.getGroupsAdmin(userItem.grouplist); @@ -89,9 +84,14 @@ const checkUserVC = async (username, vcname) => { return userVCs.includes(vcname); }; -const checkUserStorageConfig = async (username, storagConfigName) => { - const userStorageConfigs = await getUserStorageConfigs(username); - return userStorageConfigs.includes(storagConfigName); +const getUserStorages = async (username) => { + const userItem = await getUser(username); + return groupModel.getGroupsStorages(userItem.grouplist); +}; + +const checkUserStorage = async (username, storageName) => { + const userStorages = await getUserStorages(username); + return userStorages.includes(storageName); }; // module exports @@ -107,6 +107,6 @@ module.exports = { getUserVCs, checkAdmin, batchUpdateUsers, - getUserStorageConfigs, - checkUserStorageConfig, + getUserStorages, + checkUserStorage, }; diff --git a/src/rest-server/src/routes/v2/index.js b/src/rest-server/src/routes/v2/index.js index ff07e01ec6..0aadf66ab5 100644 --- a/src/rest-server/src/routes/v2/index.js +++ b/src/rest-server/src/routes/v2/index.js @@ -38,6 +38,7 @@ router.use('/user', userRouter); router.use('/group', groupRouter); -router.use('/storage', storageRouter); +router.use('/storages', storageRouter); + // module exports module.exports = router; diff --git a/src/rest-server/src/routes/v2/storage.js b/src/rest-server/src/routes/v2/storage.js index 536c7455ab..628015170f 100644 --- a/src/rest-server/src/routes/v2/storage.js +++ b/src/rest-server/src/routes/v2/storage.js @@ -17,60 +17,15 @@ // module dependencies const express = require('express'); -const storageController = require('@pai/controllers/v2/storage'); const token = require('@pai/middlewares/token'); -// const storageInputSchema = require('@pai/config/v2/storage'); +const controller = require('@pai/controllers/v2/storage'); const router = new express.Router(); -router - .route('/server/:name') - /** Get /api/v2/storage/server/:name */ - .get(token.check, storageController.getStorageServer); +router.route('/') + .get(token.check, controller.list); -router - .route('/server') - /** Get /api/v2/storage/server */ - .get(token.check, storageController.getStorageServers); - -router - .route('/server') - /** Post /api/v2/storage/server */ - .post(token.check, storageController.createStorageServer); - -router - .route('/server') - /** Put /api/v2/storage/server */ - .put(token.check, storageController.updateStorageServer); - -router - .route('/server/:name') - /** Post /api/v2/storage/server/delete */ - .delete(token.check, storageController.deleteStorageServer); - -router - .route('/config/:name') - /** Get /api/v2/storage/config/:name */ - .get(token.check, storageController.getStorageConfig); - -router - .route('/config') - /** Get /api/v2/storage/config */ - .get(token.check, storageController.getStorageConfigs); - -router - .route('/config') - /** Post /api/v2/storage/config */ - .post(token.check, storageController.createStorageConfig); - -router - .route('/config') - /** Put /api/v2/storage/config */ - .put(token.check, storageController.updateStorageConfig); - -router - .route('/config/:name') - /** Delete /api/v2/storage/config/:name */ - .delete(token.check, storageController.deleteStorageConfig); +router.route('/:storageName') + .get(token.check, controller.get); module.exports = router; diff --git a/src/rest-server/src/utils/dbUtil.js b/src/rest-server/src/utils/dbUtil.js deleted file mode 100644 index 872ce3f3fd..0000000000 --- a/src/rest-server/src/utils/dbUtil.js +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// module dependencies -const LocalCache = require('./localCache'); -const UserSecret = require('./userSecret'); - -const getStorageObject = (type, options = null) => { - switch (type) { - case 'localCache': - return new LocalCache(options); - case 'UserSecret': - return new UserSecret(options); - default: - } -}; - -module.exports = {getStorageObject}; diff --git a/src/rest-server/src/utils/error.d.ts b/src/rest-server/src/utils/error.d.ts index f64610600d..01d5d5e997 100644 --- a/src/rest-server/src/utils/error.d.ts +++ b/src/rest-server/src/utils/error.d.ts @@ -40,6 +40,7 @@ declare type Code = 'NoJobSshInfoError' | 'NoUserError' | 'NoGroupError' | + 'NoStorageError' | 'NoVirtualClusterError' | 'ReadOnlyJobError' | 'RemoveAdminError' | diff --git a/src/rest-server/src/utils/localCache.js b/src/rest-server/src/utils/localCache.js deleted file mode 100644 index b796c4bfe1..0000000000 --- a/src/rest-server/src/utils/localCache.js +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -const NodeCache = require('node-cache'); - -const StorageBase = require('./storageBase'); - -class LocalCache extends StorageBase { - constructor(options) { - super(); - let ttl = options && options.ttlSeconds ? options.ttlSeconds : 0; - let period = Math.max(ttl / 2, 60); - this.store = new NodeCache({ - stdTTL: options && options.ttlSeconds ? options.ttlSeconds : 0, - period: period, - }); - } - - get(key, options, callback) { - this.store.get(key, function(err, val) { - if (err) { - callback(err, null); - } else { - callback(null, !val && options && options.defaultValue ? options.defaultValue : val); - } - }); - } - - set(key, value, options, callback) { - let handler = function(err, success) { - if (!err && !success) { - err = new Error(`Failed to set value for key "${key}".`); - } - if (err) { - callback(err, null); - } else { - callback(null, true); - } - }; - if (options && options.ttlSeconds) { - this.store.set(key, value, options.ttlSeconds, handler); - } else { - this.store.set(key, value, handler); - } - } - - delete(key, options, callback) { - this.store.del(key, function(err, count) { - if (err) { - callback(err, null); - } else { - callback(null, options && options.checkExistence ? count == 1 : true); - } - }); - } - - has(key, options, callback) { - this.store.get(key, function(err, val) { - if (err) { - callback(err, null); - } else { - let res = val != undefined; - if (options.ignoreNull) { - res = (val != null) && res; - } - callback(null, res); - } - }); - } -} - -module.exports = LocalCache; diff --git a/src/rest-server/src/utils/storageBase.js b/src/rest-server/src/utils/storageBase.js deleted file mode 100644 index 34d32416ba..0000000000 --- a/src/rest-server/src/utils/storageBase.js +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// module dependencies - - -class StorageBase { - get(key, options, callback) { } - - set(key, value, options, callback) { } - - delete(key, options, callback) { } - - has(key, options, callback) { } -} - -module.exports = StorageBase; diff --git a/src/rest-server/src/utils/userSecret.js b/src/rest-server/src/utils/userSecret.js deleted file mode 100644 index 9fd24d82db..0000000000 --- a/src/rest-server/src/utils/userSecret.js +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// module dependencies -const {getClient} = require('@pai/models/kubernetes/kubernetes'); -const StorageBase = require('./storageBase'); - -class UserSecret extends StorageBase { - constructor(options) { - super(); - this.request = getClient(`/api/v1/namespaces/${options.paiUserNameSpace}/secrets`); - this.options = options; - } - - async get(key, options) { - try { - const hexKey = key ? Buffer.from(key).toString('hex') : ''; - const response = await this.request.get(`/${hexKey}`, { - headers: { - 'Accept': 'application/json', - }, - }); - let allUserSecrets = []; - let userData = response['data']; - if (userData.hasOwnProperty('items')) { - userData['items'].forEach((item) => { - allUserSecrets.push({ - username: Buffer.from(item['data']['username'], 'base64').toString(), - password: Buffer.from(item['data']['password'], 'base64').toString(), - admin: Buffer.from(item['data']['admin'], 'base64').toString(), - virtualCluster: item['data'].hasOwnProperty('virtualCluster') ? Buffer.from(item['data']['virtualCluster'], 'base64').toString() : 'default', - githubPAT: item['data'].hasOwnProperty('githubPAT') ? Buffer.from(item['data']['githubPAT'], 'base64').toString() : '', - }); - }); - } else { - allUserSecrets.push({ - username: Buffer.from(userData['data']['username'], 'base64').toString(), - password: Buffer.from(userData['data']['password'], 'base64').toString(), - admin: Buffer.from(userData['data']['admin'], 'base64').toString(), - virtualCluster: userData['data'].hasOwnProperty('virtualCluster') ? Buffer.from(userData['data']['virtualCluster'], 'base64').toString() : 'default', - githubPAT: userData['data'].hasOwnProperty('githubPAT') ? Buffer.from(userData['data']['githubPAT'], 'base64').toString() : '', - }); - } - return allUserSecrets; - } catch (error) { - throw error.response; - } - } - - async set(key, value, options) { - try { - const hexKey = key ? Buffer.from(key).toString('hex') : ''; - let userData = { - 'metadata': {'name': hexKey}, - 'data': { - 'username': Buffer.from(value['username']).toString('base64'), - 'password': Buffer.from(value['password']).toString('base64'), - 'admin': Buffer.from(value['admin'].toString()).toString('base64'), - }, - }; - if (value.hasOwnProperty('virtualCluster')) { - userData['data']['virtualCluster'] = Buffer.from(value['virtualCluster']).toString('base64'); - } - if (value.hasOwnProperty('githubPAT')) { - userData['data']['githubPAT'] = Buffer.from(value['githubPAT']).toString('base64'); - } - let response = null; - if (options && options['update']) { - response = await this.request.put(`/${hexKey}`, userData); - } else { - response = await this.request.post('', userData); - } - return response; - } catch (error) { - throw error.response; - } - } - - async delete(key, options) { - try { - const hexKey = key ? Buffer.from(key).toString('hex') : ''; - let response = await this.request.delete(`/${hexKey}`, { - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }, - }); - return response; - } catch (error) { - throw error.response; - } - } - - async prepareBasePath() { - try { - const response = await this.request.post('', { - 'metadata': {'name': `${this.options.paiUserNameSpace}`}, - }); - return response; - } catch (error) { - throw error.response; - } - } - - async checkBasePath() { - try { - const response = await this.request.get(`${this.options.paiUserNameSpace}`, { - headers: { - 'Accept': 'application/json', - }, - }); - return response; - } catch (error) { - throw error.response; - } - } -} -module.exports = UserSecret; From e2dbb74d31b066d521befcbb0473c0188b557c20 Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Mon, 20 Jan 2020 17:39:44 +0800 Subject: [PATCH 02/12] Fix lint Fix lint. --- src/rest-server/src/controllers/v2/storage.js | 2 +- src/rest-server/src/models/v2/job/k8s.js | 4 +- src/rest-server/test/k8sSecret.js | 312 ------------------ 3 files changed, 3 insertions(+), 315 deletions(-) delete mode 100644 src/rest-server/test/k8sSecret.js diff --git a/src/rest-server/src/controllers/v2/storage.js b/src/rest-server/src/controllers/v2/storage.js index 12640c3927..36fc89282b 100644 --- a/src/rest-server/src/controllers/v2/storage.js +++ b/src/rest-server/src/controllers/v2/storage.js @@ -28,7 +28,7 @@ const list = asyncHandler(async (req, res) => { }); const get = asyncHandler(async (req, res) => { - const storageName = req.params.storageName + const storageName = req.params.storageName; const userName = req.user.username; const admin = req.user.admin; const data = await storage.get(storageName, admin ? undefined : userName); diff --git a/src/rest-server/src/models/v2/job/k8s.js b/src/rest-server/src/models/v2/job/k8s.js index c996cfd2b6..17f8e948c6 100644 --- a/src/rest-server/src/models/v2/job/k8s.js +++ b/src/rest-server/src/models/v2/job/k8s.js @@ -579,7 +579,7 @@ const generateTaskRole = (frameworkName, taskRole, jobInfo, frameworkEnvList, co frameworkTaskRole.task.pod.spec.containers[0].volumeMounts.push({ name: `${storage.name}-volume`, mountPath: storage.mountPath || `/mnt/${storage.name}`, - ...storage.share === false && {subPath: `${jobInfo.userName}`}, + ...(storage.share === false) && {subPath: jobInfo.userName}, }); frameworkTaskRole.task.pod.spec.volumes.push({ name: `${storage.name}-volume`, @@ -897,7 +897,7 @@ const put = async (frameworkName, config, rawConfig) => { if ('extras' in config && config.extras.storages) { const userStorages = {}; (await storageModel.list()).storages - .forEach(userStorage => userStorages[userStorage.name] = userStorage); + .forEach((userStorage) => userStorages[userStorage.name] = userStorage); for (let storage of config.extras.storages) { if (!storage.name) { continue; diff --git a/src/rest-server/test/k8sSecret.js b/src/rest-server/test/k8sSecret.js deleted file mode 100644 index d8d31b428c..0000000000 --- a/src/rest-server/test/k8sSecret.js +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// test -const util = require('util'); -const dbUtility = require('@pai/utils/dbUtil'); - -const db = dbUtility.getStorageObject('UserSecret', { - 'paiUserNameSpace': 'pai-user', -}); -describe('k8s secret get function test', () => { - afterEach(function() { - if (!nock.isDone()) { - // TODO: Revamp this file and enable the following error. - // this.test.error(new Error('Not all nock interceptors were used!')); - nock.cleanAll(); - } - }); - - beforeEach(() => { - // Mock for case1 return all userinfo - nock(apiServerRootUri) - .get('/api/v1/namespaces/pai-user/secrets/') - .reply(200, { - 'kind': 'SecretList', - 'apiVersion': 'v1', - 'metadata': { - 'selfLink': '/api/v1/namespaces/pai-user/secrets/', - 'resourceVersion': '1062682', - }, - 'items': [ - { - 'metadata': { - 'name': 'cantest001', - }, - 'data': { - 'admin': 'ZmFsc2U=', - 'password': 'OGRiYjYyMWEwYWY0Y2NhMDk3NTU5MmJkNzQ0M2NkNzc5YzRkYjEwMzA2NGExYTE1MWI4YjAyYmNkZjJkYmEwNjBlMzFhNTRhYzI4MjJlYjZmZTY0ZTgxM2ZkODg0MzI5ZjNiYTYwMGFlNmQ2NjMzNGYwYjhkYzIwYTIyM2MzOWU=', - 'username': 'Y2FudGVzdDAwMQ==', - 'virtualCluster': 'ZGVmYXVsdA==', - }, - 'type': 'Opaque', - }, - { - 'metadata': { - 'name': 'pai_test', - }, - 'data': { - 'admin': 'dHJ1ZQ==', - 'password': 'MzFhNzQ0YzNhZjg5MDU2MDI0ZmY2MmMzNTZmNTQ3ZGRjMzUzYWQ3MjdkMzEwYTc3MzcxODgxMjk4MmQ1YzZlZmMzYmZmNzBkYjVlMTA0M2JkMjFkMmVkYzg4M2M4Y2Q0ZjllNzRhMWU1MjA1NDMzNjQ5MzYxMTQ4YmE4OTY0MzQ=', - 'username': 'cGFpdGVzdA==', - 'virtualCluster': 'ZGVmYXVsdCx2YzIsdmMz', - }, - 'type': 'Opaque', - }, - ], - }); - - // mock for case3 username=paitest - nock(apiServerRootUri) - .get('/api/v1/namespaces/pai-user/secrets/70616974657374') - .reply(200, { - 'kind': 'Secret', - 'apiVersion': 'v1', - 'metadata': { - 'name': '70616974657374', - }, - 'data': { - 'admin': 'dHJ1ZQ==', - 'password': 'MzFhNzQ0YzNhZjg5MDU2MDI0ZmY2MmMzNTZmNTQ3ZGRjMzUzYWQ3MjdkMzEwYTc3MzcxODgxMjk4MmQ1YzZlZmMzYmZmNzBkYjVlMTA0M2JkMjFkMmVkYzg4M2M4Y2Q0ZjllNzRhMWU1MjA1NDMzNjQ5MzYxMTQ4YmE4OTY0MzQ=', - 'username': 'cGFpdGVzdA==', - 'virtualCluster': 'ZGVmYXVsdCx2YzIsdmMz', - }, - 'type': 'Opaque', - }); - - - // mock for case2 username=non_exist - nock(apiServerRootUri) - .get('/api/v1/namespaces/pai-user/secrets/6e6f6e5f6578697374') - .reply(404, { - 'kind': 'Status', - 'apiVersion': 'v1', - 'metadata': {}, - 'status': 'Failure', - 'message': 'secrets \'6e6f6e5f6578697374\' not found', - 'reason': 'NotFound', - 'details': { - 'name': 'nonexist', - 'kind': 'secrets', - }, - 'code': 404, - }); - }); - - - // positive test case - // get exist single key value pair - it('should return whole user list', (done) => { - const dbGet = util.callbackify(db.get.bind(db)); - dbGet('', null, (err, res) => { - expect(res).to.have.lengthOf(2); - done(); - }); - }); - - // negative test case - // get non-exist user - it('should report user not found error', (done) => { - const dbGet = util.callbackify(db.get.bind(db)); - dbGet('non_exist', null, (err, res) => { - expect(err.status).to.be.equal(404); - done(); - }); - }); - - // positive test case - // find specific user - it('should return specific user info', (done) => { - const dbGet = util.callbackify(db.get.bind(db)); - dbGet('paitest', null, (err, res) => { - expect(res).to.have.lengthOf(1); - expect(res).to.have.deep.members([{ - username: 'paitest', - password: '31a744c3af89056024ff62c356f547ddc353ad727d310a773718812982d5c6efc3bff70db5e1043bd21d2edc883c8cd4f9e74a1e5205433649361148ba896434', - admin: 'true', - virtualCluster: 'default,vc2,vc3', - githubPAT: '', - }]); - done(); - }); - }); -}); - -describe('k8s secret set function test', () => { - afterEach(function() { - if (!nock.isDone()) { - // TODO: Revamp this file and enable the following error. - // this.test.error(new Error('Not all nock interceptors were used!')); - nock.cleanAll(); - } - }); - - beforeEach(() => { - // Mock for case2 username=existuser - nock(apiServerRootUri) - .put('/api/v1/namespaces/pai-user/secrets/657869737475736572', { - 'metadata': {'name': '657869737475736572'}, - 'data': { - 'admin': 'ZmFsc2U=', - 'password': 'cGFpNjY2', - 'username': 'ZXhpc3R1c2Vy', - }, - }) - .reply(200, { - 'kind': 'Secret', - 'apiVersion': 'v1', - 'metadata': { - 'name': '657869737475736572', - 'namespace': 'pai-user', - 'selfLink': '/api/v1/namespaces/pai-user/secrets/657869737475736572', - 'uid': 'd5d686ff-f9c6-11e8-b564-000d3ab5296b', - 'resourceVersion': '1115478', - 'creationTimestamp': '2018-12-07T02:21:42Z', - }, - 'data': { - 'admin': 'ZmFsc2U=', - 'password': 'cGFpNjY2', - 'username': 'ZXhpc3R1c2Vy', - }, - 'type': 'Opaque', - }); - - // Mock for case2 username=newuser - nock(apiServerRootUri) - .post('/api/v1/namespaces/pai-user/secrets', { - 'metadata': {'name': '6e657775736572'}, - 'data': { - 'admin': 'ZmFsc2U=', - 'password': 'cGFpNjY2', - 'username': 'bmV3dXNlcg==', - }, - }) - .reply(200, { - 'kind': 'Secret', - 'apiVersion': 'v1', - 'metadata': { - 'name': '6e657775736572', - 'namespace': 'pai-user', - 'selfLink': '/api/v1/namespaces/pai-user/secrets/6e657775736572', - 'uid': 'f75b6065-f9c7-11e8-b564-000d3ab5296b', - 'resourceVersion': '1116114', - 'creationTimestamp': '2018-12-07T02:29:47Z', - }, - 'data': { - 'admin': 'ZmFsc2U=', - 'password': 'cGFpNjY2', - 'username': 'bmV3dXNlcg==', - }, - 'type': 'Opaque', - }); - }); - - // set a key value pair - it('should add a new user', (done) => { - const dbSet = util.callbackify(db.set.bind(db)); - const updateUser = { - 'username': 'newuser', - 'password': 'pai666', - 'admin': false, - 'modify': false, - }; - dbSet('newuser', updateUser, null, (err, res) => { - expect(err).to.be.null; - expect(res, 'status').to.have.status(200); - done(); - }); - }); - - // update a user - it('should update an exist new user', (done) => { - const dbSet = util.callbackify(db.set.bind(db)); - const updateUser = { - 'username': 'existuser', - 'password': 'pai666', - 'admin': false, - 'modify': false, - }; - const options = {'update': true}; - dbSet('existuser', updateUser, options, (err, res) => { - expect(err).to.be.null; - expect(res, 'status').to.have.status(200); - done(); - }); - }); -}); - -describe('k8s secret delete function test', () => { - afterEach(function() { - if (!nock.isDone()) { - // TODO: Revamp this file and enable the following error. - // this.test.error(new Error('Not all nock interceptors were used!')); - nock.cleanAll(); - } - }); - - beforeEach(() => { - // Mock for case1 username=existuser - nock(apiServerRootUri) - .delete('/api/v1/namespaces/pai-user/secrets/657869737475736572') - .reply(200, { - 'kind': 'Status', - 'apiVersion': 'v1', - 'metadata': {}, - 'status': 'Success', - 'details': { - 'name': '657869737475736572', - 'kind': 'secrets', - 'uid': 'd5d686ff-f9c6-11e8-b564-000d3ab5296b', - }, - }); - - // Mock for case2 username=nonexistuser - nock(apiServerRootUri) - .delete('/api/v1/namespaces/pai-user/secrets/6e6f6e657869737475736572') - .reply(404, { - 'kind': 'Status', - 'apiVersion': 'v1', - 'metadata': {}, - 'status': 'Failure', - 'message': 'secrets \'6e6f6e657869737475736572\' not found', - 'reason': 'NotFound', - 'details': { - 'name': '6e6f6e657869737475736572', - 'kind': 'secrets', - }, - 'code': 404, - }); - }); - - // delete exist user - it('should delete an exist user successfully', (done) => { - const dbDelete = util.callbackify(db.delete.bind(db)); - dbDelete('existuser', (err, res) => { - expect(err).to.be.null; - expect(res, 'status').to.have.status(200); - done(); - }); - }); - - it('should failed to delete an non-exist user', (done) => { - const dbDelete = util.callbackify(db.delete.bind(db)); - dbDelete('nonexistuser', (err, res) => { - expect(err, 'status').to.have.status(404); - done(); - }); - }); -}); From 9bbc9520a95941824003367660bc7da79ca14fa9 Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Fri, 21 Feb 2020 19:55:01 +0800 Subject: [PATCH 03/12] Update * Add secret data into storage * Backward compatibility for * GET /storage/config * GET /storage/server * Update api docs --- src/rest-server/docs/swagger.yaml | 11 ++ .../src/controllers/v2/storage-deprecated.js | 139 ++++++++++++++++++ src/rest-server/src/models/v2/group.js | 18 +-- src/rest-server/src/models/v2/storage.js | 16 ++ src/rest-server/src/models/v2/user.js | 4 +- src/rest-server/src/routes/v2/index.js | 2 + .../src/routes/v2/storage-deprecated.js | 37 +++++ 7 files changed, 215 insertions(+), 12 deletions(-) create mode 100644 src/rest-server/src/controllers/v2/storage-deprecated.js create mode 100644 src/rest-server/src/routes/v2/storage-deprecated.js diff --git a/src/rest-server/docs/swagger.yaml b/src/rest-server/docs/swagger.yaml index 361d65e8cf..f2be59cc9e 100644 --- a/src/rest-server/docs/swagger.yaml +++ b/src/rest-server/docs/swagger.yaml @@ -1650,9 +1650,20 @@ components: - samba - azureFile - azureBlob + - other - unknown data: type: object + properties: + server: string + path: string + address: string + username: string + password: string + shareName: string + containerName: string + accountName: string + accountKey: string secretName: type: string mountOptions: diff --git a/src/rest-server/src/controllers/v2/storage-deprecated.js b/src/rest-server/src/controllers/v2/storage-deprecated.js new file mode 100644 index 0000000000..7cc3720a2c --- /dev/null +++ b/src/rest-server/src/controllers/v2/storage-deprecated.js @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation +// All rights reserved. +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +// to permit persons to whom the Software is furnished to do so, subject to the following conditions: +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// module dependencies +const asyncHandler = require('@pai/middlewares/v2/asyncHandler'); +const {get, list} = require('@pai/models/v2/storage'); +const {getUserStorages} = require('@pai/models/v2/user'); + + +const convertConfig = (storage, userDefaultStorages) => { + const config = { + name: storage.name, + default: (storage.name in userDefaultStorages), + servers: [storage.volumeName], + mountInfos: [], + }; + if (storage.share === 'false') { + config.mountInfos = [ + { + mountPoint: '/data', + path: 'data', + server: storage.volumeName, + permission: 'rw', + }, + { + mountPoint: '/home', + path: 'users/${PAI_USER_NAME}', + server: storage.volumeName, + permission: 'rw', + }, + ]; + } else { + config.mountInfos = [ + { + mountPoint: `/mnt/${storage.name}`, + path: 'data', + server: storage.volumeName, + permission: 'rw', + }, + ]; + } + return config; +}; + +const convertServer = async (storage) => { + const detail = await get(storage.name); + const server = { + spn: storage.volumeName, + type: detail.type.toLowerCase(), + data: {}, + extension: {}, + }; + if (server.type === 'nfs') { + server.data = { + address: detail.data.server, + rootPath: detail.data.path, + }; + } else if (server.type === 'samba') { + const address = detail.data.address.replace(/^\/\//, ''); + server.data = { + address: address.substr(0, address.indexOf('/')), + rootPath: address.substr(1 + address.indexOf('/')), + userName: detail.data.username, + password: detail.data.password, + domain: '', + }; + } else if (server.type === 'azurefile') { + server.data = { + dataStore: `${detail.data.accountName}.file.core.windows.net`, + fileShare: detail.data.shareName, + accountName: detail.data.accountName, + key: detail.data.accountKey, + }; + } else if (server.type === 'azureblob') { + server.data = { + dataStore: '', + containerName: detail.data.containerName, + accountName: detail.data.accountName, + key: detail.data.accountKey, + }; + } + return server; +}; + +const getConfig = asyncHandler(async (req, res) => { + let name = null; + if (req.params.name) { + name = req.params.name; + } else if (req.query.names) { + name = req.query.names; + } + + const userName = req.user.username; + const admin = req.user.admin; + const userDefaultStorages = await getUserStorages(userName, true); + const storages = (await list(admin ? undefined : userName)).storages + .filter((item) => name ? item.name === name : true) + .map((item) => convertConfig(item, userDefaultStorages)); + + res.json(storages); +}); + +const getServer = asyncHandler(async (req, res) => { + let name = null; + if (req.params.name) { + name = req.params.name; + } else if (req.query.names) { + name = req.query.names; + } + + const userName = req.user.username; + const admin = req.user.admin; + const storages = await Promise.all( + (await list(admin ? undefined : userName)).storages + .filter((item) => name ? item.volumeName === name : true) + .map(convertServer)); + + res.json(storages); +}); + +// module exports +module.exports = { + getConfig, + getServer, +}; diff --git a/src/rest-server/src/models/v2/group.js b/src/rest-server/src/models/v2/group.js index ee51011790..ef70bcc642 100644 --- a/src/rest-server/src/models/v2/group.js +++ b/src/rest-server/src/models/v2/group.js @@ -100,26 +100,25 @@ const getGroupsVCs = async (grouplist) => { return getVCsWithGroupInfo(groupItems); }; -const getStorageConfigsWithGroupInfo = async (groupItems) => { +const getStorageConfigsWithGroupInfo = async (groupItems, filterDefault=false) => { let storageConfigs = new Set(); for (const groupItem of groupItems) { if (groupItem.extension && groupItem.extension.acls) { if (groupItem.extension.acls.storageConfigs) { - storageConfigs = new Set([...storageConfigs, ...groupItem.extension.acls.storageConfigs]); + if (filterDefault) { + storageConfigs.add(groupItem.extension.acls.storageConfigs[0]); + } else { + storageConfigs = new Set([...storageConfigs, ...groupItem.extension.acls.storageConfigs]); + } } } } return [...storageConfigs]; }; -const getGroupStorages = async (groupname) => { - const groupItem = await getGroup(groupname); - return getStorageConfigsWithGroupInfo([groupItem]); -}; - -const getGroupsStorages = async (grouplist) => { +const getGroupsStorages = async (grouplist, filterDefault=false) => { const groupItems = await getListGroup(grouplist); - return getStorageConfigsWithGroupInfo(groupItems); + return getStorageConfigsWithGroupInfo(groupItems, filterDefault); }; const getAdminWithGroupInfo = (groupItems) => { @@ -489,7 +488,6 @@ module.exports = { getGroupVCs, getGroupsVCs, getVCsWithGroupInfo, - getGroupStorages, getGroupsStorages, getStorageConfigsWithGroupInfo, filterExistGroups, diff --git a/src/rest-server/src/models/v2/storage.js b/src/rest-server/src/models/v2/storage.js index c5363ba85f..084827f843 100644 --- a/src/rest-server/src/models/v2/storage.js +++ b/src/rest-server/src/models/v2/storage.js @@ -19,6 +19,7 @@ const status = require('statuses'); const createError = require('@pai/utils/error'); const user = require('@pai/models/v2/user'); +const secret = require('@pai/models/kubernetes/k8s-secret'); const kubernetes = require('@pai/models/kubernetes/kubernetes'); @@ -91,6 +92,21 @@ const convertVolumeDetail = async (pvc) => { storage.type = 'unknown'; storage.data = {}; } + + if (storage.secretName) { + const secretData = await secret.get('default', storage.secretName); + if (storage.type === 'azureFile') { + storage.data.accountName = secretData.azurestorageaccountname; + storage.data.accountKey = secretData.azurestorageaccountkey; + } else if (storage.type === 'azureBlob') { + storage.data.accountName = secretData.accountname; + storage.data.accountKey = secretData.accountkey; + } else if (storage.type === 'samba') { + storage.data.username = secretData.username; + storage.data.password = secretData.password; + } + } + return storage; }; diff --git a/src/rest-server/src/models/v2/user.js b/src/rest-server/src/models/v2/user.js index 489c454c5f..bd5fb6504b 100644 --- a/src/rest-server/src/models/v2/user.js +++ b/src/rest-server/src/models/v2/user.js @@ -84,9 +84,9 @@ const checkUserVC = async (username, vcname) => { return userVCs.includes(vcname); }; -const getUserStorages = async (username) => { +const getUserStorages = async (username, filterDefault=false) => { const userItem = await getUser(username); - return groupModel.getGroupsStorages(userItem.grouplist); + return groupModel.getGroupsStorages(userItem.grouplist, filterDefault); }; const checkUserStorage = async (username, storageName) => { diff --git a/src/rest-server/src/routes/v2/index.js b/src/rest-server/src/routes/v2/index.js index 0aadf66ab5..27abf919ff 100644 --- a/src/rest-server/src/routes/v2/index.js +++ b/src/rest-server/src/routes/v2/index.js @@ -21,6 +21,7 @@ const express = require('express'); const userRouter = require('@pai/routes/v2/user'); const groupRouter = require('@pai/routes/v2/group'); const storageRouter = require('@pai/routes/v2/storage'); +const storageDeprecatedRouter = require('@pai/routes/v2/storage-deprecated'); const controller = require('@pai/controllers/v2'); const jobRouter = require('@pai/routes/v2/job'); const virtualClusterRouter = require('@pai/routes/v2/virtual-cluster'); @@ -39,6 +40,7 @@ router.use('/user', userRouter); router.use('/group', groupRouter); router.use('/storages', storageRouter); +router.use('/storage', storageDeprecatedRouter); // module exports module.exports = router; diff --git a/src/rest-server/src/routes/v2/storage-deprecated.js b/src/rest-server/src/routes/v2/storage-deprecated.js new file mode 100644 index 0000000000..2cd2c9d941 --- /dev/null +++ b/src/rest-server/src/routes/v2/storage-deprecated.js @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation +// All rights reserved. +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +// to permit persons to whom the Software is furnished to do so, subject to the following conditions: +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// module dependencies +const express = require('express'); +const token = require('@pai/middlewares/token'); +const controller = require('@pai/controllers/v2/storage-deprecated'); + +const router = new express.Router(); + +router.route('/config') + .get(token.check, controller.getConfig); + +router.route('/config/:name') + .get(token.check, controller.getConfig); + +router.route('/server') + .get(token.check, controller.getServer); + +router.route('/server/:name') + .get(token.check, controller.getServer); + +module.exports = router; From ab6c336c574fd8d218329d5aa3379cba265bb55f Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Wed, 26 Feb 2020 17:17:24 +0800 Subject: [PATCH 04/12] Update * fix default config issue * fix multiple configs in url query * resolve comments --- .../src/controllers/v2/storage-deprecated.js | 32 ++++++++++++------- src/rest-server/src/models/v2/group.js | 4 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/rest-server/src/controllers/v2/storage-deprecated.js b/src/rest-server/src/controllers/v2/storage-deprecated.js index 7cc3720a2c..580009b5d3 100644 --- a/src/rest-server/src/controllers/v2/storage-deprecated.js +++ b/src/rest-server/src/controllers/v2/storage-deprecated.js @@ -24,11 +24,11 @@ const {getUserStorages} = require('@pai/models/v2/user'); const convertConfig = (storage, userDefaultStorages) => { const config = { name: storage.name, - default: (storage.name in userDefaultStorages), + default: (userDefaultStorages.includes(storage.name)), servers: [storage.volumeName], mountInfos: [], }; - if (storage.share === 'false') { + if (storage.share === false) { config.mountInfos = [ { mountPoint: '/data', @@ -47,7 +47,7 @@ const convertConfig = (storage, userDefaultStorages) => { config.mountInfos = [ { mountPoint: `/mnt/${storage.name}`, - path: 'data', + path: '', server: storage.volumeName, permission: 'rw', }, @@ -67,7 +67,9 @@ const convertServer = async (storage) => { if (server.type === 'nfs') { server.data = { address: detail.data.server, - rootPath: detail.data.path, + rootPath: detail.share === false ? + detail.data.path.replace(/\/users\/?$/, '') : + detail.data.path, }; } else if (server.type === 'samba') { const address = detail.data.address.replace(/^\/\//, ''); @@ -97,36 +99,42 @@ const convertServer = async (storage) => { }; const getConfig = asyncHandler(async (req, res) => { - let name = null; + let names = null; if (req.params.name) { - name = req.params.name; + names = req.params.name; } else if (req.query.names) { - name = req.query.names; + names = req.query.names; + } + if (typeof names === 'string') { + names = [names]; } const userName = req.user.username; const admin = req.user.admin; const userDefaultStorages = await getUserStorages(userName, true); const storages = (await list(admin ? undefined : userName)).storages - .filter((item) => name ? item.name === name : true) + .filter((item) => names ? names.includes(item.name) : true) .map((item) => convertConfig(item, userDefaultStorages)); res.json(storages); }); const getServer = asyncHandler(async (req, res) => { - let name = null; + let names = null; if (req.params.name) { - name = req.params.name; + names = req.params.name; } else if (req.query.names) { - name = req.query.names; + names = req.query.names; + } + if (typeof names === 'string') { + names = [names]; } const userName = req.user.username; const admin = req.user.admin; const storages = await Promise.all( (await list(admin ? undefined : userName)).storages - .filter((item) => name ? item.volumeName === name : true) + .filter((item) => names ? names.includes(item.volumeName) : true) .map(convertServer)); res.json(storages); diff --git a/src/rest-server/src/models/v2/group.js b/src/rest-server/src/models/v2/group.js index ef70bcc642..6cece3ffe6 100644 --- a/src/rest-server/src/models/v2/group.js +++ b/src/rest-server/src/models/v2/group.js @@ -101,14 +101,14 @@ const getGroupsVCs = async (grouplist) => { }; const getStorageConfigsWithGroupInfo = async (groupItems, filterDefault=false) => { - let storageConfigs = new Set(); + const storageConfigs = new Set(); for (const groupItem of groupItems) { if (groupItem.extension && groupItem.extension.acls) { if (groupItem.extension.acls.storageConfigs) { if (filterDefault) { storageConfigs.add(groupItem.extension.acls.storageConfigs[0]); } else { - storageConfigs = new Set([...storageConfigs, ...groupItem.extension.acls.storageConfigs]); + groupItem.extension.acls.storageConfigs.forEach(storageConfigs.add, storageConfigs); } } } From c5722cc6153b68688e84695617c836e4bb1871cf Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Wed, 26 Feb 2020 18:48:01 +0800 Subject: [PATCH 05/12] Update api docs Update swagger api docs. --- src/rest-server/docs/swagger.yaml | 55 ++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/src/rest-server/docs/swagger.yaml b/src/rest-server/docs/swagger.yaml index f2be59cc9e..c1d9345877 100644 --- a/src/rest-server/docs/swagger.yaml +++ b/src/rest-server/docs/swagger.yaml @@ -1653,24 +1653,55 @@ components: - other - unknown data: - type: object - properties: - server: string - path: string - address: string - username: string - password: string - shareName: string - containerName: string - accountName: string - accountKey: string + oneOf: + - type: object + description: nfs type + properties: + server: + type: string + path: + type: string + required: [server, path] + - type: object + description: samba type + properties: + address: + type: string + username: + type: string + password: + type: string + required: [address] + - type: object + description: azureFile type + properties: + shareName: + type: string + accountName: + type: string + accountKey: + type: string + required: [shareName] + - type: object + description: azureBlob type + properties: + containerName: + type: string + accountName: + type: string + accountKey: + type: string + required: [containerName] + - type: object + description: other/unknown type + properties: null secretName: type: string mountOptions: type: array items: type: string - required: [name, share, volumeName] + required: [name, share, volumeName, type, data] JobAttempt: type: object description: TODO From 9a933a41ee59f528c8465c77297298c92c5943da Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Thu, 27 Feb 2020 11:46:34 +0800 Subject: [PATCH 06/12] Update Fix params and query difference in backward compatibility. --- .../src/controllers/v2/storage-deprecated.js | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/rest-server/src/controllers/v2/storage-deprecated.js b/src/rest-server/src/controllers/v2/storage-deprecated.js index 580009b5d3..20cae1a833 100644 --- a/src/rest-server/src/controllers/v2/storage-deprecated.js +++ b/src/rest-server/src/controllers/v2/storage-deprecated.js @@ -17,6 +17,7 @@ // module dependencies const asyncHandler = require('@pai/middlewares/v2/asyncHandler'); +const createError = require('@pai/utils/error'); const {get, list} = require('@pai/models/v2/storage'); const {getUserStorages} = require('@pai/models/v2/user'); @@ -116,7 +117,15 @@ const getConfig = asyncHandler(async (req, res) => { .filter((item) => names ? names.includes(item.name) : true) .map((item) => convertConfig(item, userDefaultStorages)); - res.json(storages); + if (req.params.name) { + if (storages.length === 1) { + res.json(storages[0]); + } else { + throw createError(500, 'UnknownError', 'Config not found.'); + } + } else { + res.json(storages); + } }); const getServer = asyncHandler(async (req, res) => { @@ -137,7 +146,15 @@ const getServer = asyncHandler(async (req, res) => { .filter((item) => names ? names.includes(item.volumeName) : true) .map(convertServer)); - res.json(storages); + if (req.params.name) { + if (storages.length === 1) { + res.json(storages[0]); + } else { + throw createError(500, 'UnknownError', 'Server not found.'); + } + } else { + res.json(storages); + } }); // module exports From 21448f95fcca7cf472e8c18e26311f4d1ef6bd73 Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Fri, 28 Feb 2020 15:41:21 +0800 Subject: [PATCH 07/12] Add default storages and check deprecated config Add default storages and check deprecated config. --- src/rest-server/src/models/v2/job/k8s.js | 52 +++++++++++++++++++----- src/rest-server/src/models/v2/storage.js | 4 +- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/rest-server/src/models/v2/job/k8s.js b/src/rest-server/src/models/v2/job/k8s.js index 17f8e948c6..9e8ed9573e 100644 --- a/src/rest-server/src/models/v2/job/k8s.js +++ b/src/rest-server/src/models/v2/job/k8s.js @@ -893,19 +893,49 @@ const put = async (frameworkName, config, rawConfig) => { throw createError('Forbidden', 'ForbiddenUserError', `User ${userName} is not allowed to do operation in ${virtualCluster}`); } + // check deprecated storages config + if ( + 'extras' in config && + !config.extras.storages && + 'com.microsoft.pai.runtimeplugin' in config.extras + ) { + for (let plugin of config.extras['com.microsoft.pai.runtimeplugin']) { + if ( + plugin.plugin === 'teamwise_storage' && + 'parameters' in plugin && + plugin.parameters.storageConfigNames + ) { + config.extras.storages = + plugin.parameters.storageConfigNames.map((name) => { + return {name}; + }); + } + } + } // check storages for current user if ('extras' in config && config.extras.storages) { - const userStorages = {}; - (await storageModel.list()).storages - .forEach((userStorage) => userStorages[userStorage.name] = userStorage); - for (let storage of config.extras.storages) { - if (!storage.name) { - continue; - } - if (!(storage.name in userStorages)) { - throw createError('Not Found', 'NoStorageError', `Storage ${storage.name} is not found.`); - } else { - storage.share = userStorages[storage.name].share; + // add default storages if config is empty + if (config.extras.storages.length === 0) { + (await storageModel.list(userName, true)).storages + .forEach((userStorage) => { + config.extras.storages.push({ + name: userStorage.name, + share: userStorage.share, + }); + }); + } else { + const userStorages = {}; + (await storageModel.list(userName)).storages + .forEach((userStorage) => userStorages[userStorage.name] = userStorage); + for (let storage of config.extras.storages) { + if (!storage.name) { + continue; + } + if (!(storage.name in userStorages)) { + throw createError('Not Found', 'NoStorageError', `Storage ${storage.name} is not found.`); + } else { + storage.share = userStorages[storage.name].share; + } } } } diff --git a/src/rest-server/src/models/v2/storage.js b/src/rest-server/src/models/v2/storage.js index 084827f843..9df94e81d0 100644 --- a/src/rest-server/src/models/v2/storage.js +++ b/src/rest-server/src/models/v2/storage.js @@ -110,7 +110,7 @@ const convertVolumeDetail = async (pvc) => { return storage; }; -const list = async (userName) => { +const list = async (userName, filterDefault=false) => { let response; try { response = await kubernetes.getClient().get( @@ -127,7 +127,7 @@ const list = async (userName) => { throw createError(response.status, 'UnknownError', response.data.message); } - const userStorages = userName ? await user.getUserStorages(userName) : undefined; + const userStorages = userName ? await user.getUserStorages(userName, filterDefault) : undefined; const storages = response.data.items .filter((item) => item.status.phase === 'Bound') .filter((item) => userStorages === undefined || userStorages.includes(item.metadata.name)) From fbdd6e92dc1fa6ae3720f5b758049cdcf230e6d8 Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Fri, 28 Feb 2020 16:07:15 +0800 Subject: [PATCH 08/12] Update mount path Update mount path. --- .../src/controllers/v2/storage-deprecated.js | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/src/rest-server/src/controllers/v2/storage-deprecated.js b/src/rest-server/src/controllers/v2/storage-deprecated.js index 20cae1a833..ab5203ab00 100644 --- a/src/rest-server/src/controllers/v2/storage-deprecated.js +++ b/src/rest-server/src/controllers/v2/storage-deprecated.js @@ -27,33 +27,13 @@ const convertConfig = (storage, userDefaultStorages) => { name: storage.name, default: (userDefaultStorages.includes(storage.name)), servers: [storage.volumeName], - mountInfos: [], + mountInfos: [{ + mountPoint: `/mnt/${storage.name}`, + path: storage.share === false ? '${PAI_USER_NAME}' : '', + server: storage.volumeName, + permission: 'rw', + }], }; - if (storage.share === false) { - config.mountInfos = [ - { - mountPoint: '/data', - path: 'data', - server: storage.volumeName, - permission: 'rw', - }, - { - mountPoint: '/home', - path: 'users/${PAI_USER_NAME}', - server: storage.volumeName, - permission: 'rw', - }, - ]; - } else { - config.mountInfos = [ - { - mountPoint: `/mnt/${storage.name}`, - path: '', - server: storage.volumeName, - permission: 'rw', - }, - ]; - } return config; }; @@ -68,9 +48,7 @@ const convertServer = async (storage) => { if (server.type === 'nfs') { server.data = { address: detail.data.server, - rootPath: detail.share === false ? - detail.data.path.replace(/\/users\/?$/, '') : - detail.data.path, + rootPath: detail.data.path, }; } else if (server.type === 'samba') { const address = detail.data.address.replace(/^\/\//, ''); From 31cdae5c663a04a05162f2265e04226967d2e1b6 Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Fri, 28 Feb 2020 17:08:09 +0800 Subject: [PATCH 09/12] Update * support storage config in NNI * add accountsastoken for blobfuse flexvolume --- src/rest-server/docs/swagger.yaml | 2 ++ src/rest-server/src/models/v2/job/k8s.js | 18 +++++++++--------- src/rest-server/src/models/v2/storage.js | 6 +++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/rest-server/docs/swagger.yaml b/src/rest-server/docs/swagger.yaml index c1d9345877..8ebbea0969 100644 --- a/src/rest-server/docs/swagger.yaml +++ b/src/rest-server/docs/swagger.yaml @@ -1691,6 +1691,8 @@ components: type: string accountKey: type: string + accountSASToken: + type: string required: [containerName] - type: object description: other/unknown type diff --git a/src/rest-server/src/models/v2/job/k8s.js b/src/rest-server/src/models/v2/job/k8s.js index 9e8ed9573e..64a72a2ded 100644 --- a/src/rest-server/src/models/v2/job/k8s.js +++ b/src/rest-server/src/models/v2/job/k8s.js @@ -900,15 +900,15 @@ const put = async (frameworkName, config, rawConfig) => { 'com.microsoft.pai.runtimeplugin' in config.extras ) { for (let plugin of config.extras['com.microsoft.pai.runtimeplugin']) { - if ( - plugin.plugin === 'teamwise_storage' && - 'parameters' in plugin && - plugin.parameters.storageConfigNames - ) { - config.extras.storages = - plugin.parameters.storageConfigNames.map((name) => { - return {name}; - }); + if (plugin.plugin === 'teamwise_storage') { + if ('parameters' in plugin && plugin.parameters.storageConfigNames) { + config.extras.storages = + plugin.parameters.storageConfigNames.map((name) => { + return {name}; + }); + } else { + config.extras.storages = []; + } } } } diff --git a/src/rest-server/src/models/v2/storage.js b/src/rest-server/src/models/v2/storage.js index 9df94e81d0..2c05059eb9 100644 --- a/src/rest-server/src/models/v2/storage.js +++ b/src/rest-server/src/models/v2/storage.js @@ -100,7 +100,11 @@ const convertVolumeDetail = async (pvc) => { storage.data.accountKey = secretData.azurestorageaccountkey; } else if (storage.type === 'azureBlob') { storage.data.accountName = secretData.accountname; - storage.data.accountKey = secretData.accountkey; + if (secretData.accountkey) { + storage.data.accountKey = secretData.accountkey; + } else if (secretData.accountsastoken) { + storage.data.accountSASToken = secretData.accountsastoken; + } } else if (storage.type === 'samba') { storage.data.username = secretData.username; storage.data.password = secretData.password; From 91020a861e66799eaac3bcf21d84e2c1f5f86392 Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Tue, 3 Mar 2020 11:14:18 +0800 Subject: [PATCH 10/12] Remove unused code Remove unused code. --- .../utils/manager/storage/crudK8sSecret.js | 400 ------------------ .../src/utils/manager/storage/crudUtil.js | 27 -- .../src/utils/manager/storage/storage.js | 120 ------ 3 files changed, 547 deletions(-) delete mode 100644 src/rest-server/src/utils/manager/storage/crudK8sSecret.js delete mode 100644 src/rest-server/src/utils/manager/storage/crudUtil.js delete mode 100644 src/rest-server/src/utils/manager/storage/storage.js diff --git a/src/rest-server/src/utils/manager/storage/crudK8sSecret.js b/src/rest-server/src/utils/manager/storage/crudK8sSecret.js deleted file mode 100644 index 70c741ec37..0000000000 --- a/src/rest-server/src/utils/manager/storage/crudK8sSecret.js +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -const Storage = require('./storage'); -const logger = require('@pai/config/logger'); -const createError = require('@pai/utils/error'); -const k8sModel = require('@pai/models/kubernetes/kubernetes'); - -const STORAGE_NAMESPACE = process.env.PAI_STORAGE_NAMESPACE || 'pai-storage'; - - -/** - * @typedef StorageServer - * @property {string} ServerInstance.spn - server name - * @property {string} ServerInstance.type - server type - * @property {Object} ServerInstance.data - server data - */ - -/** - * @typedef StorageConfig - * @property {string} ConfigInstance.name - config name - * @property {boolean} ConfigInstance.default - config type - * @property {Array} ConfigInstance.servers - config data - * @property {Array} ConfigInstance.mountInfos - config data - */ - -/** - * @function readStorageServer - return a Storage Server's info based on spn. - * @async - * @param {string} key - Server spn - * @return {Promise} A promise to the StorageServer instance - */ -async function readStorageServer(key) { - try { - const request = k8sModel.getClient('/api/v1/namespaces'); - - logger.info(`${STORAGE_NAMESPACE}/secrets/storage-server`); - const response = await request.get( - `${STORAGE_NAMESPACE}/secrets/storage-server`, - { - headers: { - Accept: 'application/json', - }, - } - ); - logger.info(response); - let serverData = JSON.parse( - Buffer.from(response['data']['data'][key], 'base64').toString() - ); - let innerData = Object.assign({}, serverData); - delete innerData.spn; - delete innerData.type; - - let serverInstance = Storage.createStorageServer({ - spn: serverData['spn'], - type: serverData['type'], - data: innerData, - extension: - serverData['extension'] !== undefined ? serverData['extension'] : {}, - }); - return serverInstance; - } catch (error) { - if (error.response) { - throw error.response; - } else { - throw error; - } - } -} - -/** - * @function readStorageServers - return Storage Server's infos based on spns. - * @async - * @param {string[]} keys - An array of server spns. If array is empty, return all StorageServers. - * @return {Promise} A promise to the StorageServer instances - */ -async function readStorageServers(keys) { - try { - const request = k8sModel.getClient('/api/v1/namespaces'); - const response = await request.get( - `${STORAGE_NAMESPACE}/secrets/storage-server`, - { - headers: { - Accept: 'application/json', - }, - } - ); - - let serverInstances = []; - if (keys.length == 0) { - keys = Object.keys(response['data']['data']); - } - - for (const key of keys) { - if (key !== 'empty' && response['data']['data'].hasOwnProperty(key)) { - let serverData = JSON.parse( - Buffer.from(response['data']['data'][key], 'base64').toString() - ); - let innerData = Object.assign({}, serverData); - delete innerData.spn; - delete innerData.type; - - serverInstances.push( - Storage.createStorageServer({ - spn: serverData['spn'], - type: serverData['type'], - data: innerData, - extension: - serverData['extension'] !== undefined - ? serverData['extension'] - : {}, - }) - ); - } - } - - return serverInstances; - } catch (error) { - if (error.response) { - throw error.response; - } else { - throw error; - } - } -} - -/** - * @function readStorageConfig - return a Storage Config's info based on config name. - * @async - * @param {string} key - Config name - * @return {Promise} A promise to the StorageConfig instance - */ -async function readStorageConfig(key) { - try { - const request = k8sModel.getClient('/api/v1/namespaces'); - const response = await request.get( - `${STORAGE_NAMESPACE}/secrets/storage-config`, - { - headers: { - Accept: 'application/json', - }, - } - ); - let configData = JSON.parse( - Buffer.from(response['data']['data'][key], 'base64').toString() - ); - let configInstance = Storage.createStorageConfig({ - name: configData['name'], - default: configData['default'], - servers: configData['servers'], - mountInfos: configData['mountInfos'], - }); - return configInstance; - } catch (error) { - if (error.response) { - throw error.response; - } else { - throw error; - } - } -} - -/** - * @function readStorageConfigs - return Storage Configs's infos based on config names. - * @async - * @param {string[]} keys - An array of config names. If array is empty, return all StorageConfigs. - * @return {Promise} A promise to the StorageConfig instances - */ -async function readStorageConfigs(keys) { - try { - const request = k8sModel.getClient('/api/v1/namespaces'); - const response = await request.get( - `${STORAGE_NAMESPACE}/secrets/storage-config`, - { - headers: { - Accept: 'application/json', - }, - } - ); - - let configInstances = []; - if (keys.length == 0) { - keys = Object.keys(response['data']['data']); - } - - for (const key of keys) { - if (key !== 'empty' && response['data']['data'].hasOwnProperty(key)) { - let configData = JSON.parse( - Buffer.from(response['data']['data'][key], 'base64').toString() - ); - configInstances.push( - Storage.createStorageConfig({ - name: configData['name'], - default: configData['default'], - servers: configData['servers'], - mountInfos: configData['mountInfos'], - }) - ); - } - } - - return configInstances; - } catch (error) { - if (error.response) { - throw error.response; - } else { - throw error; - } - } -} - -/** - * @function patchStorageServer - Patch Storage Server entry to kubernetes secrets. - * @async - * @param {string} op - Patch operation type, could be add, replace or remove - * @param {string} key - Storage Server name - * @param {StorageServer} value - Storage Server info, should be null when op is remove - * @return {Promise} A promise to patch result. - */ -async function patchStorageServer(op, key, value) { - if (key === 'empty') { - throw createError('Forbidden', 'ForbiddenKeyError', 'Key \'empty\' is system reserved and should not be modified!'); - } - - try { - const request = k8sModel.getClient('/api/v1/namespaces'); - let serverData = { - op: op, - path: `/data/${key}`, - }; - if (value !== null) { - let serverInstance = Storage.createStorageServer(value); - serverData.value = Buffer.from( - JSON.stringify({ - spn: serverInstance['spn'], - type: serverInstance['type'], - ...serverInstance['data'], - extension: - serverInstance['extension'] !== undefined - ? serverInstance['extension'] - : {}, - }) - ).toString('base64'); - } - logger.info(serverData); - return await request.patch( - `${STORAGE_NAMESPACE}/secrets/storage-server`, - [serverData], - { - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json-patch+json', - }, - } - ); - } catch (error) { - logger.debug(`Error when ${op} StorageServer ${key}, please check.`); - if (error.response) { - throw error.response; - } else { - throw error; - } - } -} - -/** - * @function patchStorageConfig - Patch Storage Config entry to kubernetes secrets. - * @async - * @param {string} op - Patch operation type, could be add, replace or remove - * @param {string} key - Storage Config name - * @param {StorageServer} value - Storage Config info, should be null when op is remove - * @return {Promise} A promise to patch result. - */ -async function patchStorageConfig(op, key, value) { - if (key === 'empty') { - throw createError('Forbidden', 'ForbiddenKeyError', 'Key \'empty\' is system reserved and should not be modified!'); - } - - try { - const request = k8sModel.getClient('/api/v1/namespaces'); - let configData = { - op: op, - path: `/data/${key}`, - }; - if (value !== null) { - let configInstance = Storage.createStorageConfig(value); - configData.value = Buffer.from(JSON.stringify(configInstance)).toString( - 'base64' - ); - } - return await request.patch( - `${STORAGE_NAMESPACE}/secrets/storage-config`, - [configData], - { - headers: { - 'Accept': 'application/json', - 'Content-Type': 'application/json-patch+json', - }, - } - ); - } catch (error) { - if (error.response) { - logger.debug(`Error when ${op} StorageConfig ${key}, please check.`); - throw error.response; - } else { - throw error; - } - } -} - -/** - * @function createStorageServer - Create a Storage Server entry to kubernetes secrets. - * @async - * @param {string} key - Storage Server name - * @param {StorageServer} value - Storage Server info - * @return {Promise} A promise to patch result. - */ -async function createStorageServer(key, value) { - return await patchStorageServer('add', key, value); -} - -/** - * @function createStorageConfig - Create a Storage Server entry to kubernetes secrets. - * @async - * @param {string} key - Storage config name - * @param {StorageConfig} value - Storage config info - * @return {Promise} A promise to patch result. - */ -async function createStorageConfig(key, value) { - return await patchStorageConfig('add', key, value); -} - -/** - * @function updateStorageServer - Update a Storage Server entry to kubernetes secrets. - * @async - * @param {string} key - Stroage server name - * @param {StorageServer} value - Stroage server info - * @return {Promise} A promise to patch result. - */ -async function updateStorageServer(key, value) { - return await patchStorageServer('replace', key, value); -} - -/** - * @function updateStorageConfig - Update a Storage Config entry to kubernetes secrets. - * @async - * @param {string} key - Stroage Config name - * @param {StorageConfig} value - Stroage Config info - * @return {Promise} A promise to patch result. - */ -async function updateStorageConfig(key, value) { - return await patchStorageConfig('replace', key, value); -} - -/** - * @function removeStorageServer - Remove a Storage Server entry from kubernetes secrets. - * @async - * @param {string} key - Storage Server name - * @return {Promise} A promise to patch result. - */ -async function removeStorageServer(key) { - return await patchStorageServer('remove', key, null); -} - -/** - * @function removeStorageConfig - Remove a Storage Config entry from kubernetes secrets. - * @async - * @param {string} key - Storage Config name - * @return {Promise} A promise to patch result. - */ -async function removeStorageConfig(key) { - return await patchStorageConfig('remove', key, null); -} - -module.exports = { - createStorageServer, - createStorageConfig, - readStorageServer, - readStorageServers, - readStorageConfig, - readStorageConfigs, - updateStorageServer, - updateStorageConfig, - removeStorageServer, - removeStorageConfig, -}; diff --git a/src/rest-server/src/utils/manager/storage/crudUtil.js b/src/rest-server/src/utils/manager/storage/crudUtil.js deleted file mode 100644 index 4e0fa7139f..0000000000 --- a/src/rest-server/src/utils/manager/storage/crudUtil.js +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -// module dependencies -const crudK8sSecret = require('./crudK8sSecret'); - -const getStorageObject = (type) => { - if (type === 'k8sSecret') { - return crudK8sSecret; - } -}; - -module.exports = {getStorageObject}; diff --git a/src/rest-server/src/utils/manager/storage/storage.js b/src/rest-server/src/utils/manager/storage/storage.js deleted file mode 100644 index 49df4c204f..0000000000 --- a/src/rest-server/src/utils/manager/storage/storage.js +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) Microsoft Corporation -// All rights reserved. -// -// MIT License -// -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -// to permit persons to whom the Software is furnished to do so, subject to the following conditions: -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING -// BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -const Joi = require('joi'); - -const storageServerSchema = Joi.object() - .keys({ - spn: Joi.string() - .regex(/^[A-Za-z0-9_]+$/, 'spn') - .required(), - type: Joi.string() - .valid(['nfs', 'samba', 'azurefile', 'azureblob', 'hdfs', 'other']) - .required(), - data: Joi.alternatives() - .when('type', { - is: 'nfs', - then: Joi.object({ - address: Joi.string().required(), - rootPath: Joi.string().required(), - }).required(), - }) - .when('type', { - is: 'samba', - then: Joi.object({ - address: Joi.string().required(), - rootPath: Joi.string().required(), - userName: Joi.string().required(), - password: Joi.string().required(), - domain: Joi.string().required(), - }).required(), - }) - .when('type', { - is: 'azurefile', - then: Joi.object({ - dataStore: Joi.string().required(), - fileShare: Joi.string().required(), - accountName: Joi.string().required(), - key: Joi.string().required(), - }).required(), - }) - .when('type', { - is: 'azureblob', - then: Joi.object({ - dataStore: Joi.string().optional(), - containerName: Joi.string().required(), - accountName: Joi.string().required(), - key: Joi.string().required(), - }).required(), - }) - .when('type', { - is: 'hdfs', - then: Joi.object({ - namenode: Joi.string().required(), - port: Joi.number().required(), - }).required(), - }), - extension: Joi.object().optional(), - }) - .required(); - -function storageServerValidate(serverValue) { - const res = storageServerSchema.validate(serverValue, {allowUnknown: true}); - if (res['error']) { - throw new Error(`Storage Server schema error\n${res['error']}`); - } - return res['value']; -} - -const storageConfigSchema = Joi.object() - .keys({ - name: Joi.string() - .regex(/^[A-Za-z0-9_]+$/, 'spn') - .required(), - default: Joi.bool().default(false), - servers: Joi.array() - .items(Joi.string()) - .optional(), - gpn: Joi.string().optional(), - mountInfos: Joi.array() - .items( - Joi.object({ - mountPoint: Joi.string().required(), - server: Joi.string().required(), - path: Joi.string().required(), - permission: Joi.string() - .valid(['ro', 'rw']) - .optional() - .default('rw'), - }) - ) - .optional(), - }) - .required(); - -function storageConfigValidate(configValue) { - const res = storageConfigSchema.validate(configValue); - if (res['error']) { - throw new Error(`Storage Config schema error\n${res['error']}`); - } - return res['value']; -} - -module.exports = { - createStorageServer: storageServerValidate, - createStorageConfig: storageConfigValidate, -}; From ab8d474a8e4028eff987c0a0b4b739fd2a0b61ec Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Tue, 3 Mar 2020 12:06:49 +0800 Subject: [PATCH 11/12] Add default option in storage query Add default option in storage query. --- src/rest-server/docs/swagger.yaml | 6 ++++++ src/rest-server/src/controllers/v2/storage-deprecated.js | 6 ++---- src/rest-server/src/controllers/v2/storage.js | 7 +++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/rest-server/docs/swagger.yaml b/src/rest-server/docs/swagger.yaml index 8ebbea0969..8a36b5eafa 100644 --- a/src/rest-server/docs/swagger.yaml +++ b/src/rest-server/docs/swagger.yaml @@ -1326,6 +1326,12 @@ paths: operationId: getStorages security: - bearerAuth: [] + parameters: + - name: default + in: query + description: Return default storage or not. + schema: + type: boolean responses: 200: description: Succeeded diff --git a/src/rest-server/src/controllers/v2/storage-deprecated.js b/src/rest-server/src/controllers/v2/storage-deprecated.js index ab5203ab00..84feef70ed 100644 --- a/src/rest-server/src/controllers/v2/storage-deprecated.js +++ b/src/rest-server/src/controllers/v2/storage-deprecated.js @@ -89,9 +89,8 @@ const getConfig = asyncHandler(async (req, res) => { } const userName = req.user.username; - const admin = req.user.admin; const userDefaultStorages = await getUserStorages(userName, true); - const storages = (await list(admin ? undefined : userName)).storages + const storages = (await list(userName)).storages .filter((item) => names ? names.includes(item.name) : true) .map((item) => convertConfig(item, userDefaultStorages)); @@ -118,9 +117,8 @@ const getServer = asyncHandler(async (req, res) => { } const userName = req.user.username; - const admin = req.user.admin; const storages = await Promise.all( - (await list(admin ? undefined : userName)).storages + (await list(userName)).storages .filter((item) => names ? names.includes(item.volumeName) : true) .map(convertServer)); diff --git a/src/rest-server/src/controllers/v2/storage.js b/src/rest-server/src/controllers/v2/storage.js index 36fc89282b..a2a2c2474a 100644 --- a/src/rest-server/src/controllers/v2/storage.js +++ b/src/rest-server/src/controllers/v2/storage.js @@ -22,16 +22,15 @@ const storage = require('@pai/models/v2/storage'); const list = asyncHandler(async (req, res) => { const userName = req.user.username; - const admin = req.user.admin; - const data = await storage.list(admin ? undefined : userName); + const filterDefault = Boolean(req.query.default); + const data = await storage.list(userName, filterDefault); res.json(data); }); const get = asyncHandler(async (req, res) => { const storageName = req.params.storageName; const userName = req.user.username; - const admin = req.user.admin; - const data = await storage.get(storageName, admin ? undefined : userName); + const data = await storage.get(storageName, userName); res.json(data); }); From c6c30ffb8e668c0715575dc101a85a1e421f536f Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Wed, 4 Mar 2020 16:20:45 +0800 Subject: [PATCH 12/12] Add document on setting up storage Add document on setting up storage. --- contrib/storage_plugin/README.MD | 17 +- docs/setup-persistent-volumes-on-pai.md | 206 ++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 7 deletions(-) create mode 100644 docs/setup-persistent-volumes-on-pai.md diff --git a/contrib/storage_plugin/README.MD b/contrib/storage_plugin/README.MD index 7d071eeefa..47ab4e89d0 100644 --- a/contrib/storage_plugin/README.MD +++ b/contrib/storage_plugin/README.MD @@ -1,11 +1,14 @@ # Team wise storage +*NOTICE: This tool has been deprecated, please refer to [Setup Kubernetes Persistent Volumes as Storage on PAI](../../docs/setup-persistent-volumes-on-pai.md).* + + A tool to manage external storage in PAI. ## Index - [ What is team wise storage](#Team_storage) - [ Team wise storage usages ](#Usages) - - [ Setup server ](#Usages_setup_server) + - [ Setup server ](#Usages_setup_server) - [ Create storage server in PAI ](#Usages_server) - [ Create storage config in PAI ](#Usages_config) - [ Set storage config access for group ](#Usages_groupsc) @@ -34,9 +37,9 @@ Team wise storage solution offers: ### Setup server - NFS -Edit /etc/exports, export /root/path/to/share +Edit /etc/exports, export /root/path/to/share ``` -/root/path/to/share (rw, sync, no_root_squash) +/root/path/to/share (rw, sync, no_root_squash) ``` no_root_squash is needed for storage plugin to creae folders. @@ -58,7 +61,7 @@ Create Azurefile share through azure web portal. Create Azureblob share through azure web portal. -### Create storage server in PAI +### Create storage server in PAI In PAI dev-box, swith to folder pai/contrib/storage-plugin Create server config using command: @@ -78,16 +81,16 @@ python storagectl.py server set NAME azurefile DATASTORE FILESHARE ACCOUNTNAME K ``` - Azureblob: -``` +``` python storagectl.py server set NAME azureblob DATASTORE CONTAINERNAME ACCOUNTNAME KEY ``` - HDFS: -``` +``` python storagectl.py server set NAME hdfs NAMENODE PORT ``` -### Create storage config in PAI +### Create storage config in PAI In PAI dev-box, swith to folder pai/contrib/storage-plugin Create config using command: diff --git a/docs/setup-persistent-volumes-on-pai.md b/docs/setup-persistent-volumes-on-pai.md new file mode 100644 index 0000000000..76aa971513 --- /dev/null +++ b/docs/setup-persistent-volumes-on-pai.md @@ -0,0 +1,206 @@ +# Setup Kubernetes Persistent Volumes as Storage on PAI + +This document describes how to use Kubernetes Persistent Volumes (PV) as storage on PAI. + +A pure k8s version PAI v0.18.0 cluster or later is required before you start. + + +## Introduction + +To use existing storage (nfs, samba, Azure blob, etc.) on PAI, admin could create PV on Kubernetes and claim as PAI storage. Then users could use those PV/PVC as storage in their jobs. +Here's a detailed walkthrough. + + +## Create PV/PVC on Kubernetes + +Admin need to create PV for storage and create PVC bound to corresponding PV. +The name of PVC is used to onboard on PAI. + +There're many approches to create PV/PVC, you could refer to [Kubernetes docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) if you are not familiar yet. Followings are some commonly used PV/PVC examples. + +* NFS + + ```yaml + # NFS Persistent Volume + apiVersion: v1 + kind: PersistentVolume + metadata: +   name: nfs-storage-pv +   labels: +     name: nfs-storage + spec: +   capacity: +     storage: 10Gi +   volumeMode: Filesystem +   accessModes: +     - ReadWriteMany +   persistentVolumeReclaimPolicy: Retain +   mountOptions: +     - nfsvers=4.1 +   nfs: +     path: /data +     server: 10.0.0.1 + --- + # NFS Persistent Volume Claim + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: +   name: nfs-storage + # labels: + # share: "false" # to mount sub path on PAI + spec: +   accessModes: +     - ReadWriteMany +   volumeMode: Filesystem +   resources: +     requests: +       storage: 10Gi # no more than PV capacity +   selector: +     matchLabels: +       name: nfs-storage # corresponding to PV label + ``` + + Save the above file as `nfs-storage.yaml` and run `kubectl apply -f nfs-storage.yaml` to create a PV named `nfs-storage-pv` and a PVC named `nfs-storage` for nfs server `nfs://10.0.0.1:/data`. The PVC will be bound to specific PV through label selector, using label `name: nfs-storage`. + + Users could use PVC name `nfs-storage` as storage name to mount this nfs storage in their jobs. + + If you want to configure the above nfs as personal storage so that each user could only visit their own directory on PAI like Linux home directory, for example, Alice can only mount `/data/Alice` while Bob can only mount `/data/Bob`, you could add a `share: "false"` label to PVC. In this case, PAI will use `${PAI_USER_NAME}` as sub path when mounting to job containers. + +* Samba + + Please refer to [this document](https://github.com/Azure/kubernetes-volume-drivers/blob/master/flexvolume/smb/README.md) to install cifs/smb FlexVolume driver and create PV/PVC for Samba. + +* Azure Blob + + Please refer to [this document](https://github.com/Azure/kubernetes-volume-drivers/blob/master/flexvolume/blobfuse/README.md) to install blobfuse FlexVolume driver and create PV/PVC for Azure Blob. + +* Azure File + + First create a Kubernetes secret to access the Azure file share. + + ```sh + kubectl create secret generic azure-secret --from-literal=azurestorageaccountname=$AKS_PERS_STORAGE_ACCOUNT_NAME --from-literal=azurestorageaccountkey=$STORAGE_KEY + ``` + + Then create PV/PVC for the file azure. + + ```yaml + # Azure File Persistent Volume + apiVersion: v1 + kind: PersistentVolume + metadata: + name: azure-file-storage-pv + labels: + name: azure-file-storage + spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteMany + storageClassName: azurefile + azureFile: + secretName: azure-secret + shareName: aksshare + readOnly: false + mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1000 + - gid=1000 + - mfsymlinks + - nobrl + --- + # Azure File Persistent Volume Claim + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: azure-file-storage + spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile + resources: + requests: + storage: 5Gi + selector: + matchLabels: +     name: azure-file-storage + ``` + + More details on Azure File volume could be found in [this document](https://docs.microsoft.com/en-us/azure/aks/azure-files-volume). + + +## Assign Storage to PAI Groups + +PAI uses Kubernetes PVC name as storage name. +To use Kubernetes volumes in PAI, amdin need to assign storage to PAI groups first. + +1. Service configuration file + + For AAD mode, storage could be configured in `service-configuration.yaml` file. + + ```yaml + authentication: + ... + group-manager: + ... + grouplist: + - groupname: group1 + externalName: sg1 + extension: + acls: + admin: false + virtualClusters: ["vc1"] + storageConfigs: ["azure-file-storage"] + - groupname: group2 + externalName: sg2 + extension: + acls: + admin: false + virtualClusters: ["vc1", "vc2"] + storageConfigs: ["nfs-storage"] + ``` + + The first storage in `storageConfigs` list will be treated as default storage for this group. + +2. RESTful API + + [Group extension API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/microsoft/pai/master/src/rest-server/docs/swagger.yaml#operation/updateGroupExtension) could be used to create or update `storageConfigs` in a given group. Here's an example for request body: + + ```json + { + "acls": { + "admin": false, + "virtualClusters": ["vc1", "vc2"], + "storageConfigs": ["nfs-storage"] + } + } + ``` + + +## Use Storage on PAI + +1. Job configuration file + + To use one or more storage in job, user could specify storage names in `extras.storages` section in job configuration file: + + ```yaml + extras: + storages: + - name: nfs-storage + mountPath: /data + - name: azure-file-storage + ``` + + Their are two fields for each storage, `name` and `mountPath`. `name` refers to storage name while `mountPath` is the mount path inside job container, which has default value `/mnt/${name}` and is optional. + + ```yaml + extras: + storages: [] + ``` + + Setting it to an empty list will mount default storage for current user in the job. + +2. RESTful API + + User could use [list storage API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/microsoft/pai/master/src/rest-server/docs/swagger.yaml#operation/getStorages) to list permitted storage, or [get storage API](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/microsoft/pai/master/src/rest-server/docs/swagger.yaml#operation/getStorage) to view the detail of a given storage.