Skip to content
This repository has been archived by the owner on Nov 16, 2023. It is now read-only.

Update OpenPAI JS SDK #13

Merged
merged 2 commits into from
Jul 2, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions i18n/common.json
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@
"storage.create.folder.error": "Failed creating folder: {0}",
"storage.create.folder.success": "Folder was successfully created",
"storage.upload.status": "Uploading to storage - {0} / {1}",
"storage.upload.error": "Failed uploading to storage: {0}",
"storage.upload.success": "Files was successfully uploaded",
"storage.upload.error": "Failed uploading to storage: {0}, please make sure you already connect to the storage. error message: {1}",
"storage.upload.success": "Files {0} was successfully uploaded",
"storage.download.status": "Uploading from storage - {0} / {1}",
"storage.download.error": "Failed downloading from storage: {0}",
"storage.download.success": "Files was successfully downloaded",
Expand All @@ -95,6 +95,7 @@
"storage.dialog.label.upload-folders": "Upload Folders...",
"storage.dialog.label.download": "Download Here",
"storage.nfs.mount.point.prompt": "Please enter the path...",
"storage.nfs.mountPoint": "OpenPAI NFS storage mount point, '<clusterName>_<storageName>: <path>'",
"storage.upload.pick.prompt": "Please choose a storage...",
"hdfs.read.cancelled": "Cancelled downloading file from HDFS.",
"hdfs.write.cancelled": "Cancelled uploading file to HDFS.",
Expand Down Expand Up @@ -124,6 +125,8 @@
"job.submission.success": "Successfully submitted job.",
"job.submission.success.open": "Open job page",
"job.submission.success.remote": "Remote job",
"job.submission.submit.anyway": "Submit job anyway",
"job.submission.submit.cancel": "Cancel submission",
"job.simulation.status": "PAI: Generating the job simulation dockerfile",
"job.simulation.error": "Error occurred while generating the job simulation dockerfile: {0}",
"job.simulation.success": "Dockerfile has been generated in {0}/{1}. Please run {2} to simulate the job running. (docker is required)",
Expand Down Expand Up @@ -226,8 +229,8 @@
"storage.create.folder.error": "新建文件夹失败: {0}",
"storage.create.folder.success": "新建文件夹成功",
"storage.upload.status": "上传到存储服务器 - {0} / {1}",
"storage.upload.error": "上传时发生错误: {0}",
"storage.upload.success": "文件已被成功上传",
"storage.upload.error": "上传到存储服务器 {0} 失败, 请确保您已经连上该存储服务器。错误信息:{1}",
"storage.upload.success": "文件 {0} 已被成功上传",
"storage.download.status": "从服务器上下载 - {0} / {1}",
"storage.download.error": "下载时发生错误: {0}",
"storage.download.success": "文件下载成功",
Expand All @@ -238,6 +241,7 @@
"storage.dialog.label.upload-folders": "上传文件夹...",
"storage.dialog.label.download": "下载到这里",
"storage.nfs.mount.point.prompt": "请输入路径...",
"storage.nfs.mountPoint": "OpenPAI NFS 存储挂载点, '<集群名称>_<NFS存储名称>: <挂载路径>'",
"storage.upload.pick.prompt": "请选择一个存储……",
"hdfs.read.cancelled": "HDFS 下载操作已取消。",
"hdfs.write.cancelled": "HDFS 上传操作已取消。",
Expand Down Expand Up @@ -269,6 +273,8 @@
"job.submission.success": "任务已成功提交。",
"job.submission.success.open": "打开任务详情页面",
"job.submission.success.remote": "远程连接任务",
"job.submission.submit.anyway": "仍然提交任务",
"job.submission.submit.cancel": "取消提交任务",
"job.request.status": "PAI: 正在提交任务",
"job.simulation.status": "PAI: 正在生成模拟任务 Dockerfile",
"job.simulation.error": "生成模拟任务 Dockerfile 时发生错误:{0}",
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -656,6 +656,7 @@
},
"dependencies": {
"@azure/storage-blob": "^12.0.1",
"@microsoft/openpai-js-sdk": "^0.1.0",
"ajv": "^6.5.4",
"axios": "^0.19.2",
"fs-extra": "^7.0.0",
Expand All @@ -669,7 +670,6 @@
"lodash": "^4.17.15",
"node-rsa": "^1.0.7",
"node-yaml-parser": "^0.0.9",
"openpai-js-sdk": "microsoft/openpaisdk",
"opn": "^5.4.0",
"reflect-metadata": "^0.1.12",
"request": "^2.88.0",
Expand Down
20 changes: 15 additions & 5 deletions schemas/pai_personal_storage.schema.json
Original file line number Diff line number Diff line change
@@ -1,26 +1,36 @@
{
"type": "object",
"properties": {
"spn": {
"name": {
"type": "string",
"description": "storage server display name",
"minLength": 1
},
"type": {
"type": "string",
"enum": ["nfs", "samba", "azureFile", "azureBlob"]
},
"data": {
"type": "object",
"description": "Azure blob",
"description": "Azure blob Example",
"properties": {
"containerName": {
"type": "string"
},
"accountName": {
"type": "string"
},
"key": {
"type": "string"
"accountKey": {
"type": "string",
"description": "Please remove this line if use SAS Token."
},
"accountSASToken": {
"type": "string",
"description": "Please remove this line if use Account Key."
}
},
"additionalProperties": true
}
}
},
"additionalProperties": true
}
16 changes: 10 additions & 6 deletions src/pai/clusterManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
* @author Microsoft
*/

import { PAIV2 } from '@microsoft/openpai-js-sdk';
import { injectable } from 'inversify';
import { clone, range } from 'lodash';
import { IAuthnInfo, ILoginInfo, OpenPAIClient } from 'openpai-js-sdk';
import * as request from 'request-promise-native';
import * as vscode from 'vscode';

Expand Down Expand Up @@ -145,14 +145,14 @@ export class ClusterManager extends Singleton {

public async autoAddOIDCUserInfo(cluster: IPAICluster): Promise<void> {
try {
const client: OpenPAIClient = new OpenPAIClient({
let client: PAIV2.OpenPAIClient = new PAIV2.OpenPAIClient({
rest_server_uri: Util.fixURL(cluster.rest_server_uri, cluster.https)
});

const authnInfo: IAuthnInfo = await client.authn.info();
const clusterInfo: PAIV2.IPAIClusterInfo = await client.api.getClusterInfo();

if (authnInfo.authn_type === 'OIDC') {
const loginInfo: ILoginInfo = await login(
if (clusterInfo.authnMethod === 'OIDC') {
const loginInfo: PAIV2.ILoginInfo = await login(
`https://${cluster.rest_server_uri}`,
`https://${cluster.web_portal_uri}`,
async () => {
Expand All @@ -172,7 +172,11 @@ export class ClusterManager extends Singleton {
let clusterToken: string = loginInfo.token;

try {
const response: any = await client.authn.createApplicationToken(clusterToken);
client = new PAIV2.OpenPAIClient({
rest_server_uri: Util.fixURL(cluster.rest_server_uri, cluster.https),
token: clusterToken
});
const response: any = await client.token.createApplicationToken();
clusterToken = response.token;
} catch (error) {
console.log('Get application token fail, use user token.');
Expand Down
24 changes: 18 additions & 6 deletions src/pai/container/jobListTreeView.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
* @author Microsoft
*/

import { PAIV1, PAIV2 } from '@microsoft/openpai-js-sdk';
import { injectable } from 'inversify';
import * as request from 'request-promise-native';
import {
commands, window, workspace, Event, EventEmitter, TreeDataProvider,
TreeItem, TreeItemCollapsibleState, TreeView, WorkspaceConfiguration
Expand Down Expand Up @@ -35,7 +35,6 @@ import { Util } from '../../common/util';
import { ClusterManager } from '../clusterManager';
import { RecentJobManager } from '../recentJobManager';
import { IPAICluster, IPAIJobInfo } from '../utility/paiInterface';
import { PAIRestUri } from '../utility/paiUri';

import { ClusterNode } from './common/clusterNode';
import { FilterType, LoadingState, TreeDataType } from './common/treeDataEnum';
Expand Down Expand Up @@ -316,10 +315,23 @@ export class JobListTreeDataProvider extends Singleton implements TreeDataProvid
cluster.loadingState = LoadingState.Loading;
this.onDidChangeTreeDataEmitter.fire(cluster);
try {
cluster.jobs = await request.get(
PAIRestUri.jobs(cluster.config),
{ json: true }
);
const clusterConfig: PAIV2.IPAICluster = {
rest_server_uri: cluster.config.rest_server_uri,
token: cluster.config.token,
username: cluster.config.username,
password: cluster.config.password,
https: cluster.config.https
};
try {
// Try OpenPAI v2 api to get jobs
const client: PAIV2.OpenPAIClient = new PAIV2.OpenPAIClient(clusterConfig);
cluster.jobs = await client.job.listJobs();
} catch {
// Try OpenPAI v1 api to get jobs
const client: PAIV1.OpenPAIClient = new PAIV1.OpenPAIClient(clusterConfig);
cluster.jobs = await client.job.list();
}

cluster.loadingState = LoadingState.Finished;
this.clusterLoadError[cluster.index] = false;
} catch (e) {
Expand Down
12 changes: 6 additions & 6 deletions src/pai/container/storage/clusterStorageTreeItem.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* @author Microsoft
*/

import { IStorageConfig, IStorageServer, OpenPAIClient } from 'openpai-js-sdk';
import { PAIV2 } from '@microsoft/openpai-js-sdk';
import { TreeItemCollapsibleState } from 'vscode';

import {
Expand All @@ -21,6 +21,8 @@ import { StorageTreeNode } from '../common/treeNode';

import { TeamStorageTreeNode } from './teamStorageTreeItem';

import OpenPAIClient = PAIV2.OpenPAIClient;

/**
* PAI cluster storage tree node.
*/
Expand Down Expand Up @@ -48,11 +50,9 @@ export class ClusterStorageTreeNode extends StorageTreeNode {
password: this.cluster.password,
https: this.cluster.https
});
const storageServers: Map<string, IStorageServer> =
new Map((await client.storage.getServer()).map(server => [server.spn, server]));
const storageConfigs: IStorageConfig[] = await client.storage.getConfig();
this.children = storageConfigs.map(config =>
new TeamStorageTreeNode(config, storageServers, this.cluster, client, this));
const storageSummary: PAIV2.IStorageSummary = await client.storage.getStorages();
this.children = storageSummary.storages.map(summary =>
new TeamStorageTreeNode(summary.name, this.cluster, client, this));
} catch (e) {
Util.err('treeview.storage.error', [e.message || e]);
}
Expand Down
91 changes: 0 additions & 91 deletions src/pai/container/storage/mountPointTreeItem.ts

This file was deleted.

Loading