diff --git a/.changeset/eleven-feet-turn.md b/.changeset/eleven-feet-turn.md new file mode 100644 index 0000000000..ca54a937ac --- /dev/null +++ b/.changeset/eleven-feet-turn.md @@ -0,0 +1,6 @@ +--- +"@scow/mis-server": minor +--- + +在管理系统和门户系统中增加依赖于管理系统的集群停用功能，在数据库中新增 Cluster 表单 +**注意：停用后集群将不可用，集群所有数据不再更新。再启用后请手动同步平台数据！** \ No newline at end of file diff --git a/.changeset/great-starfishes-pump.md b/.changeset/great-starfishes-pump.md new file mode 100644 index 0000000000..c614a9fa1b --- /dev/null +++ b/.changeset/great-starfishes-pump.md @@ -0,0 +1,5 @@ +--- +"@scow/ai": patch +--- + +同步操作日志服务中的日志类型，增加启用集群，停用集群 diff --git a/.changeset/grumpy-months-cover.md b/.changeset/grumpy-months-cover.md new file mode 100644 index 0000000000..f03703ece0 --- /dev/null +++ b/.changeset/grumpy-months-cover.md @@ -0,0 +1,5 @@ +--- +"@scow/config": patch +--- + +增加集群停用功能通用类型 diff --git a/.changeset/long-kids-wash.md b/.changeset/long-kids-wash.md new file mode 100644 index 0000000000..fc3823c900 --- /dev/null +++ b/.changeset/long-kids-wash.md @@ -0,0 +1,12 @@ +--- +"@scow/portal-server": minor +"@scow/portal-web": minor +"@scow/mis-web": minor +"@scow/lib-server": minor +"@scow/cli": minor +"@scow/lib-web": minor +"@scow/docs": minor +--- + +在管理系统和门户系统中增加依赖于管理系统的集群停用功能 +**注意：停用后集群将不可用，集群所有数据不再更新。再启用后请手动同步平台数据！** diff --git a/.changeset/weak-chicken-worry.md b/.changeset/weak-chicken-worry.md new file mode 100644 index 0000000000..b4b6edf250 --- /dev/null +++ b/.changeset/weak-chicken-worry.md @@ -0,0 +1,6 @@ +--- +"@scow/grpc-api": minor +--- + +新增集群停用功能 api: getClustersRuntimeInfo, activateCluster, deactivateCluster +新增获取集群配置信息api： getClusterConfigFiles diff --git a/apps/ai/src/models/operationLog.ts b/apps/ai/src/models/operationLog.ts index d4b72274ea..d727aafe93 100644 --- a/apps/ai/src/models/operationLog.ts +++ b/apps/ai/src/models/operationLog.ts @@ -80,6 +80,8 @@ export const OperationType: OperationTypeEnum = { setAccountBlockThreshold: "setAccountBlockThreshold", setAccountDefaultBlockThreshold: "setAccountDefaultBlockThreshold", userChangeTenant: "userChangeTenant", + activateCluster: "activateCluster", + deactivateCluster: "deactivateCluster", customEvent: "customEvent", }; diff --git a/apps/cli/src/compose/index.ts b/apps/cli/src/compose/index.ts index d7d5e6941a..c42a5b8969 100644 --- a/apps/cli/src/compose/index.ts +++ b/apps/cli/src/compose/index.ts @@ -250,6 +250,8 @@ export const createComposeSpec = (config: InstallConfigSchema) => { environment: { SCOW_LAUNCH_APP: "portal-server", PORTAL_BASE_PATH: portalBasePath, + MIS_DEPLOYED: config.mis ? "true" : "false", + MIS_SERVER_URL: config.mis ? "mis-server:5000" : "", ...serviceLogEnv, ...nodeOptions ? { NODE_OPTIONS: nodeOptions } : {}, }, @@ -269,6 +271,7 @@ export const createComposeSpec = (config: InstallConfigSchema) => { "BASE_PATH": portalBasePath, "MIS_URL": join(BASE_PATH, MIS_PATH), "MIS_DEPLOYED": config.mis ? "true" : "false", + "MIS_SERVER_URL": config.mis ? "mis-server:5000" : "", "AI_URL": join(BASE_PATH, AI_PATH), "AI_DEPLOYED": config.ai ? "true" : "false", "AUTH_EXTERNAL_URL": config.auth.custom?.external?.url || join(BASE_PATH, "/auth"), diff --git a/apps/cli/tests/compose.test.ts b/apps/cli/tests/compose.test.ts index 7502e4daa4..67a8fe87bd 100644 --- a/apps/cli/tests/compose.test.ts +++ b/apps/cli/tests/compose.test.ts @@ -42,6 +42,7 @@ it("generate correct paths", async () => { const composeConfig = createComposeSpec(config); expect(composeConfig.services["portal-web"].environment).toContain("MIS_URL=/mis"); + expect(composeConfig.services["portal-web"].environment).toContain("MIS_SERVER_URL=mis-server:5000"); expect(composeConfig.services["mis-web"].environment).toContain("PORTAL_URL=/"); expect(composeConfig.services["ai"].environment).toContain("MIS_URL=/mis"); }); diff --git a/apps/mis-server/src/app.ts b/apps/mis-server/src/app.ts index 4f59a95d83..70e73939c7 100644 --- a/apps/mis-server/src/app.ts +++ b/apps/mis-server/src/app.ts @@ -42,6 +42,7 @@ export async function createServer() { for (const plugin of plugins) { await server.register(plugin); } + await server.register(accountServiceServer); await server.register(userServiceServer); await server.register(adminServiceServer); diff --git a/apps/mis-server/src/bl/PriceMap.ts b/apps/mis-server/src/bl/PriceMap.ts index ecaa43cdcf..baa696a2d4 100644 --- a/apps/mis-server/src/bl/PriceMap.ts +++ b/apps/mis-server/src/bl/PriceMap.ts @@ -15,7 +15,7 @@ import { Logger } from "@ddadaal/tsgrpc-server"; import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; import { Partition } from "@scow/scheduler-adapter-protos/build/protos/config"; import { calculateJobPrice } from "src/bl/jobPrice"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { misConfig } from "src/config/mis"; import { JobPriceInfo } from "src/entities/JobInfo"; import { AmountStrategy, JobPriceItem } from "src/entities/JobPriceItem"; @@ -90,7 +90,10 @@ export async function createPriceMap( // partitions info for all clusters const partitionsForClusters: Record = {}; + + // call for all config clusters const reply = await clusterPlugin.callOnAll( + configClusters, logger, async (client) => await asyncClientCall(client.config, "getClusterConfig", {}), ); @@ -106,10 +109,9 @@ export async function createPriceMap( const missingPaths = [] as string[]; - for (const cluster in clusters) { + for (const cluster in configClusters) { for (const partition of partitionsForClusters[cluster]) { const path = [cluster, partition.name]; - const { qos } = partition; if (path.join(".") in defaultPrices) { diff --git a/apps/mis-server/src/bl/block.ts b/apps/mis-server/src/bl/block.ts index 4e5b2e8965..3808dc3799 100644 --- a/apps/mis-server/src/bl/block.ts +++ b/apps/mis-server/src/bl/block.ts @@ -14,12 +14,16 @@ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { Logger } from "@ddadaal/tsgrpc-server"; import { Loaded } from "@mikro-orm/core"; import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; import { BlockedFailedUserAccount } from "@scow/protos/build/server/admin"; import { Account } from "src/entities/Account"; import { UserAccount, UserStatus } from "src/entities/UserAccount"; import { ClusterPlugin } from "src/plugins/clusters"; import { callHook } from "src/plugins/hookClient"; +import { getActivatedClusters } from "./clustersUtils"; + + /** * Update block status of accounts and users in the slurm. * If it is whitelisted, it doesn't block. @@ -31,15 +35,33 @@ export async function updateBlockStatusInSlurm( ) { const blockedAccounts: string[] = []; const blockedFailedAccounts: string[] = []; + const blockedUserAccounts: [string, string][] = []; + const blockedFailedUserAccounts: BlockedFailedUserAccount[] = []; + const accounts = await em.find(Account, { blockedInCluster: true }); + const currentActivatedClusters = await getActivatedClusters(em, logger).catch((e) => { + logger.info(e); + return {}; + }); + + if (Object.keys(currentActivatedClusters).length === 0) { + logger.info("No available activated clusters in SCOW."); + return { + blockedAccounts, + blockedFailedAccounts, + blockedUserAccounts, + blockedFailedUserAccounts, + }; + } + for (const account of accounts) { if (account.whitelist) { continue; } try { - await clusterPlugin.callOnAll(logger, async (client) => + await clusterPlugin.callOnAll(currentActivatedClusters, logger, async (client) => await asyncClientCall(client.account, "blockAccount", { accountName: account.accountName, }), @@ -50,15 +72,14 @@ export async function updateBlockStatusInSlurm( } } - const blockedUserAccounts: [string, string][] = []; - const blockedFailedUserAccounts: BlockedFailedUserAccount[] = []; + const userAccounts = await em.find(UserAccount, { blockedInCluster: UserStatus.BLOCKED, }, { populate: ["user", "account"]}); for (const ua of userAccounts) { try { - await clusterPlugin.callOnAll(logger, async (client) => + await clusterPlugin.callOnAll(currentActivatedClusters, logger, async (client) => await asyncClientCall(client.user, "blockUserInAccount", { accountName: ua.account.$.accountName, userId: ua.user.$.userId, @@ -108,9 +129,22 @@ export async function updateUnblockStatusInSlurm( const unblockedAccounts: string[] = []; const unblockedFailedAccounts: string[] = []; + const currentActivatedClusters = await getActivatedClusters(em, logger).catch((e) => { + logger.info(e); + return {}; + }); + + if (Object.keys(currentActivatedClusters).length === 0) { + logger.info("No available activated clusters in SCOW."); + return { + unblockedAccounts, + unblockedFailedAccounts, + }; + } + for (const account of accounts) { try { - await clusterPlugin.callOnAll(logger, async (client) => + await clusterPlugin.callOnAll(currentActivatedClusters, logger, async (client) => await asyncClientCall(client.account, "unblockAccount", { accountName: account.accountName, }), @@ -140,7 +174,10 @@ export async function updateUnblockStatusInSlurm( * @returns Operation result **/ export async function blockAccount( - account: Loaded, clusterPlugin: ClusterPlugin["clusters"], logger: Logger, + account: Loaded, + currentActivatedClusters: Record, + clusterPlugin: ClusterPlugin["clusters"], + logger: Logger, ): Promise<"AlreadyBlocked" | "Whitelisted" | "OK"> { if (account.blockedInCluster) { return "AlreadyBlocked"; } @@ -149,7 +186,7 @@ export async function blockAccount( return "Whitelisted"; } - await clusterPlugin.callOnAll(logger, async (client) => { + await clusterPlugin.callOnAll(currentActivatedClusters, logger, async (client) => { await asyncClientCall(client.account, "blockAccount", { accountName: account.accountName, }); @@ -170,12 +207,15 @@ export async function blockAccount( * @returns Operation result **/ export async function unblockAccount( - account: Loaded, clusterPlugin: ClusterPlugin["clusters"], logger: Logger, + account: Loaded, + currentActivatedClusters: Record, + clusterPlugin: ClusterPlugin["clusters"], + logger: Logger, ): Promise<"OK" | "ALREADY_UNBLOCKED"> { if (!account.blockedInCluster) { return "ALREADY_UNBLOCKED"; } - await clusterPlugin.callOnAll(logger, async (client) => { + await clusterPlugin.callOnAll(currentActivatedClusters, logger, async (client) => { await asyncClientCall(client.account, "unblockAccount", { accountName: account.accountName, }); @@ -193,6 +233,7 @@ export async function unblockAccount( * */ export async function blockUserInAccount( ua: Loaded, + currentActivatedClusters: Record, clusterPlugin: ClusterPlugin, logger: Logger, ) { if (ua.blockedInCluster == UserStatus.BLOCKED) { @@ -202,7 +243,7 @@ export async function blockUserInAccount( const accountName = ua.account.$.accountName; const userId = ua.user.$.userId; - await clusterPlugin.clusters.callOnAll(logger, async (client) => + await clusterPlugin.clusters.callOnAll(currentActivatedClusters, logger, async (client) => await asyncClientCall(client.user, "blockUserInAccount", { accountName, userId, @@ -222,6 +263,7 @@ export async function blockUserInAccount( * */ export async function unblockUserInAccount( ua: Loaded, + currentActivatedClusters: Record, clusterPlugin: ClusterPlugin, logger: Logger, ) { if (ua.blockedInCluster === UserStatus.UNBLOCKED) { @@ -231,7 +273,7 @@ export async function unblockUserInAccount( const accountName = ua.account.getProperty("accountName"); const userId = ua.user.getProperty("userId"); - await clusterPlugin.clusters.callOnAll(logger, async (client) => + await clusterPlugin.clusters.callOnAll(currentActivatedClusters, logger, async (client) => await asyncClientCall(client.user, "unblockUserInAccount", { accountName, userId, diff --git a/apps/mis-server/src/bl/charging.ts b/apps/mis-server/src/bl/charging.ts index be390f3f50..0fccb1f43e 100644 --- a/apps/mis-server/src/bl/charging.ts +++ b/apps/mis-server/src/bl/charging.ts @@ -13,6 +13,7 @@ import { Logger } from "@ddadaal/tsgrpc-server"; import { Loaded } from "@mikro-orm/core"; import { SqlEntityManager } from "@mikro-orm/mysql"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; import { Decimal, decimalToMoney } from "@scow/lib-decimal"; import { blockAccount, blockUserInAccount, unblockAccount, unblockUserInAccount } from "src/bl/block"; import { Account } from "src/entities/Account"; @@ -58,6 +59,7 @@ export function checkShouldUnblockAccount(account: Loaded) { export async function pay( request: PayRequest, em: SqlEntityManager, + currentActivatedClusters: Record, logger: Logger, clusterPlugin: ClusterPlugin, ) { const { @@ -92,7 +94,7 @@ export async function pay( && checkShouldUnblockAccount(target) ) { logger.info("Unblock account %s", target.accountName); - await unblockAccount(target, clusterPlugin.clusters, logger); + await unblockAccount(target, currentActivatedClusters, clusterPlugin.clusters, logger); } if ( @@ -100,7 +102,7 @@ export async function pay( && checkShouldBlockAccount(target) ) { logger.info("Block account %s", target.accountName); - await blockAccount(target, clusterPlugin.clusters, logger); + await blockAccount(target, currentActivatedClusters, clusterPlugin.clusters, logger); } return { @@ -120,6 +122,7 @@ type ChargeRequest = { export async function charge( request: ChargeRequest, em: SqlEntityManager, + currentActivatedClusters: Record, logger: Logger, clusterPlugin: ClusterPlugin, ) { const { target, amount, comment, type, userId, metadata } = request; @@ -144,7 +147,7 @@ export async function charge( && checkShouldBlockAccount(target) ) { logger.info("Block account %s due to out of balance.", target.accountName); - await blockAccount(target, clusterPlugin.clusters, logger); + await blockAccount(target, currentActivatedClusters, clusterPlugin.clusters, logger); } return { @@ -155,7 +158,10 @@ export async function charge( export async function addJobCharge( ua: Loaded, - charge: Decimal, clusterPlugin: ClusterPlugin, logger: Logger, + charge: Decimal, + currentActivatedClusters: Record, + clusterPlugin: ClusterPlugin, + logger: Logger, ) { if (ua.usedJobCharge && ua.jobChargeLimit) { ua.usedJobCharge = ua.usedJobCharge.plus(charge); @@ -167,16 +173,19 @@ export async function addJobCharge( ).shouldBlockInCluster; if (shouldBlockUserInCluster) { - await blockUserInAccount(ua, clusterPlugin, logger); + await blockUserInAccount(ua, currentActivatedClusters, clusterPlugin, logger); } else { - await unblockUserInAccount(ua, clusterPlugin, logger); + await unblockUserInAccount(ua, currentActivatedClusters, clusterPlugin, logger); } } } export async function setJobCharge( ua: Loaded, - charge: Decimal, clusterPlugin: ClusterPlugin, logger: Logger, + charge: Decimal, + currentActivatedClusters: Record, + clusterPlugin: ClusterPlugin, + logger: Logger, ) { ua.jobChargeLimit = charge; if (!ua.usedJobCharge) { @@ -190,9 +199,9 @@ export async function setJobCharge( ).shouldBlockInCluster; if (shouldBlockUserInCluster) { - await blockUserInAccount(ua, clusterPlugin, logger); + await blockUserInAccount(ua, currentActivatedClusters, clusterPlugin, logger); } else { - await unblockUserInAccount(ua, clusterPlugin, logger); + await unblockUserInAccount(ua, currentActivatedClusters, clusterPlugin, logger); } } } diff --git a/apps/mis-server/src/bl/clustersUtils.ts b/apps/mis-server/src/bl/clustersUtils.ts new file mode 100644 index 0000000000..82e7d00a73 --- /dev/null +++ b/apps/mis-server/src/bl/clustersUtils.ts @@ -0,0 +1,118 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { ServiceError } from "@ddadaal/tsgrpc-common"; +import { Logger } from "@ddadaal/tsgrpc-server"; +import { status } from "@grpc/grpc-js"; +import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; +import { scowErrorMetadata } from "@scow/lib-server/build/error"; +import { NO_ACTIVATED_CLUSTERS } from "@scow/lib-server/build/misCommon/clustersActivation"; +import { ClusterActivationStatus, + clusterActivationStatusFromJSON, ClusterRuntimeInfo, + ClusterRuntimeInfo_LastActivationOperation } from "@scow/protos/build/server/config"; +import { configClusters } from "src/config/clusters"; +import { Cluster } from "src/entities/Cluster"; + + +export async function updateCluster( + em: SqlEntityManager, + configClusterIds: string[], + logger: Logger, +): Promise { + await em.transactional(async (txnEm) => { + logger.info("Update cluster entity started."); + + const clustersFromDb = await txnEm.find(Cluster, {}); + + const existingClusterIds = clustersFromDb.map((item) => item.clusterId); + + // Delete non-existent cluster IDs from the database + const shouldDeleteClusters = clustersFromDb.filter((cluster) => !configClusterIds.includes(cluster.clusterId)); + if (shouldDeleteClusters.length > 0) { + logger.info("Start Delete clusters."); + txnEm.remove(shouldDeleteClusters); + logger.info("Cluster IDs: %s not existed in the config files have been marked for deletion.", + shouldDeleteClusters.map((x) => x.clusterId)); + } + + // Write new records for new cluster IDs + const shouldCreateClusterIds = configClusterIds.filter((id) => !existingClusterIds.includes(id)); + if (shouldCreateClusterIds.length > 0) { + logger.info("Start Create clusters."); + await Promise.all( + shouldCreateClusterIds.map(async (id) => { + logger.info("To insert clusterId: %s", id); + const newCluster = new Cluster({ + clusterId: id, + }); + txnEm.persist(newCluster); + }), + ); + + logger.info("Cluster IDs: %s from config files have been created in Cluster.", shouldCreateClusterIds); + } + await txnEm.flush(); + }); + +} + +export async function getClustersRuntimeInfo( + em: SqlEntityManager, + logger: Logger, +): Promise { + + const clustersFromDb = await em.find(Cluster, {}); + + const reply = clustersFromDb.map((x) => { + + return { + clusterId: x.clusterId, + activationStatus: clusterActivationStatusFromJSON(x.activationStatus), + lastActivationOperation: x.lastActivationOperation as ClusterRuntimeInfo_LastActivationOperation, + updateTime: x.updateTime ? new Date(x.updateTime).toISOString() : "", + }; + }); + + const clusterDatabaseList = clustersFromDb.map((x) => { + return `(Cluster ID: ${x.clusterId}) : ${x.activationStatus}`; + }).join("; "); + + logger.info("Current clusters list: %s", clusterDatabaseList); + + return reply; +} + + +export const getActivatedClusters = async (em: SqlEntityManager, logger: Logger) => { + + const clustersDbInfo = await getClustersRuntimeInfo(em, logger); + + const currentActivatedClusterIds = clustersDbInfo.filter((cluster) => { + return cluster.activationStatus === ClusterActivationStatus.ACTIVATED; + }).map((cluster) => cluster.clusterId); + + if (currentActivatedClusterIds.length === 0) { + throw new ServiceError({ + code: status.INTERNAL, + details: "No available clusters. Please try again later", + metadata: scowErrorMetadata(NO_ACTIVATED_CLUSTERS, { currentActivatedClusters: "" }), + }); + } + + return currentActivatedClusterIds.reduce((acc, clusterId) => { + if (configClusters[clusterId]) { + acc[clusterId] = configClusters[clusterId]; + } + return acc; + }, {} as Record); +}; diff --git a/apps/mis-server/src/bl/importUsers.ts b/apps/mis-server/src/bl/importUsers.ts index 9fe82283e0..c44f016c61 100644 --- a/apps/mis-server/src/bl/importUsers.ts +++ b/apps/mis-server/src/bl/importUsers.ts @@ -14,6 +14,7 @@ import { Logger } from "@ddadaal/tsgrpc-server"; import { ServiceError } from "@grpc/grpc-js"; import { Status } from "@grpc/grpc-js/build/src/constants"; import { SqlEntityManager } from "@mikro-orm/mysql"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; import { blockAccount, unblockAccount } from "src/bl/block"; import { Account, AccountState } from "src/entities/Account"; import { AccountWhitelist } from "src/entities/AccountWhitelist"; @@ -34,7 +35,9 @@ export interface ImportUsersData { } export async function importUsers(data: ImportUsersData, em: SqlEntityManager, - whitelistAll: boolean, clusterPlugin: ClusterPlugin["clusters"], logger: Logger) + whitelistAll: boolean, + currentActivatedClusters: Record, + clusterPlugin: ClusterPlugin["clusters"], logger: Logger) { const tenant = await em.findOneOrFail(Tenant, { name: DEFAULT_TENANT_NAME }); @@ -129,7 +132,7 @@ export async function importUsers(data: ImportUsersData, em: SqlEntityManager, failedUnblockAccounts.push(acc.accountName); } else { try { - await unblockAccount(account, clusterPlugin, logger); + await unblockAccount(account, currentActivatedClusters, clusterPlugin, logger); } catch (e) { // 集群解锁账户失败，记录失败账户 failedUnblockAccounts.push(account.accountName); @@ -162,7 +165,7 @@ export async function importUsers(data: ImportUsersData, em: SqlEntityManager, failedBlockAccounts.push(acc.accountName); } else { try { - await blockAccount(account, clusterPlugin, logger); + await blockAccount(account, currentActivatedClusters, clusterPlugin, logger); } catch (e) { // 集群封锁账户失败，记录失败账户 failedBlockAccounts.push(account.accountName); diff --git a/apps/mis-server/src/bl/jobPrice.ts b/apps/mis-server/src/bl/jobPrice.ts index 79fd0ec7b1..4b2eaee34c 100644 --- a/apps/mis-server/src/bl/jobPrice.ts +++ b/apps/mis-server/src/bl/jobPrice.ts @@ -16,7 +16,7 @@ import { Decimal } from "@scow/lib-decimal"; import { Partition } from "@scow/scheduler-adapter-protos/build/protos/config"; import { join } from "path"; import { JobInfo, PriceMap } from "src/bl/PriceMap"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { misConfig } from "src/config/mis"; import { JobPriceInfo } from "src/entities/JobInfo"; import { AmountStrategy, JobPriceItem } from "src/entities/JobPriceItem"; @@ -74,6 +74,8 @@ export async function calculateJobPrice( logger.trace(`Calculating price for job ${info.jobId} in cluster ${info.cluster}`); + // use all clusters from config files + const clusters = configClusters; const clusterInfo = clusters[info.cluster]; if (!clusterInfo) { diff --git a/apps/mis-server/src/config/clusters.ts b/apps/mis-server/src/config/clusters.ts index 7bae580abd..bf40e17800 100644 --- a/apps/mis-server/src/config/clusters.ts +++ b/apps/mis-server/src/config/clusters.ts @@ -13,4 +13,4 @@ import { getClusterConfigs } from "@scow/config/build/cluster"; import { logger } from "src/utils/logger"; -export const clusters = getClusterConfigs(undefined, logger); +export const configClusters = getClusterConfigs(undefined, logger); diff --git a/apps/mis-server/src/entities/Cluster.ts b/apps/mis-server/src/entities/Cluster.ts new file mode 100644 index 0000000000..080978bce5 --- /dev/null +++ b/apps/mis-server/src/entities/Cluster.ts @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { Entity, Enum, PrimaryKey, Property } from "@mikro-orm/core"; +import { DATETIME_TYPE } from "src/utils/orm"; + +export enum ClusterActivationStatus { + ACTIVATED = "ACTIVATED", + DEACTIVATED = "DEACTIVATED", +} + +export interface LastActivationOperation { + // activation operator userId + operatorId?: string, + // comment only when deactivate a cluster + deactivationComment?: string, +} + +@Entity() +export class Cluster { + @PrimaryKey() + id!: number; + + @Property({ unique: true }) + clusterId: string; + + @Enum({ items: () => ClusterActivationStatus, + default: ClusterActivationStatus.ACTIVATED, comment: Object.values(ClusterActivationStatus).join(", ") }) + activationStatus: ClusterActivationStatus; + + @Property({ type: "json", nullable: true }) + lastActivationOperation?: LastActivationOperation; + + @Property({ columnType: DATETIME_TYPE, nullable: true }) + createTime: Date; + + @Property({ columnType: DATETIME_TYPE, nullable: true, onUpdate: () => new Date() }) + updateTime: Date; + + constructor(init: { + clusterId: string; + activationStatus?: ClusterActivationStatus; + lastActivationOperation?: LastActivationOperation; + createTime?: Date; + updateTime?: Date; + }) { + this.clusterId = init.clusterId; + this.activationStatus = init.activationStatus || ClusterActivationStatus.ACTIVATED; + this.lastActivationOperation = init.lastActivationOperation; + this.createTime = init.createTime ?? new Date(); + this.updateTime = init.updateTime ?? new Date(); + } +} diff --git a/apps/mis-server/src/entities/index.ts b/apps/mis-server/src/entities/index.ts index 5a547986aa..45aec9369a 100644 --- a/apps/mis-server/src/entities/index.ts +++ b/apps/mis-server/src/entities/index.ts @@ -24,6 +24,8 @@ import { Tenant } from "src/entities/Tenant"; import { User } from "src/entities/User"; import { UserAccount } from "src/entities/UserAccount"; +import { Cluster } from "./Cluster"; + export const entities = [ UserAccount, AccountWhitelist, @@ -38,4 +40,5 @@ export const entities = [ ChargeRecord, SystemState, QueryCache, + Cluster, ]; diff --git a/apps/mis-server/src/index.ts b/apps/mis-server/src/index.ts index 5450f04f97..c05b345b34 100644 --- a/apps/mis-server/src/index.ts +++ b/apps/mis-server/src/index.ts @@ -32,7 +32,7 @@ async function main() { switch (command) { case "fetchJobs": - await fetchJobs(em, logger, server.ext, server.ext); + await fetchJobs(em, logger, server.ext); break; case "createPriceItems": diff --git a/apps/mis-server/src/migrations/.snapshot-scow_server.json b/apps/mis-server/src/migrations/.snapshot-scow_server.json index 876ae3736a..34bb434d46 100644 --- a/apps/mis-server/src/migrations/.snapshot-scow_server.json +++ b/apps/mis-server/src/migrations/.snapshot-scow_server.json @@ -247,6 +247,98 @@ "foreignKeys": {}, "nativeEnums": {} }, + { + "columns": { + "id": { + "name": "id", + "type": "int", + "unsigned": true, + "autoincrement": true, + "primary": true, + "nullable": false, + "mappedType": "integer" + }, + "cluster_id": { + "name": "cluster_id", + "type": "varchar(255)", + "unsigned": false, + "autoincrement": false, + "primary": false, + "nullable": false, + "mappedType": "string" + }, + "activation_status": { + "name": "activation_status", + "type": "enum('ACTIVATED','DEACTIVATED')", + "unsigned": false, + "autoincrement": false, + "primary": false, + "nullable": false, + "default": "'ACTIVATED'", + "enumItems": [ + "ACTIVATED", + "DEACTIVATED" + ], + "comment": "ACTIVATED, DEACTIVATED", + "mappedType": "enum" + }, + "last_activation_operation": { + "name": "last_activation_operation", + "type": "json", + "unsigned": false, + "autoincrement": false, + "primary": false, + "nullable": true, + "mappedType": "json" + }, + "create_time": { + "name": "create_time", + "type": "DATETIME(6)", + "unsigned": false, + "autoincrement": false, + "primary": false, + "nullable": true, + "length": 6, + "mappedType": "datetime" + }, + "update_time": { + "name": "update_time", + "type": "DATETIME(6)", + "unsigned": false, + "autoincrement": false, + "primary": false, + "nullable": true, + "length": 6, + "mappedType": "datetime" + } + }, + "name": "cluster", + "indexes": [ + { + "columnNames": [ + "cluster_id" + ], + "composite": false, + "keyName": "cluster_cluster_id_unique", + "constraint": true, + "primary": false, + "unique": true + }, + { + "keyName": "PRIMARY", + "columnNames": [ + "id" + ], + "composite": false, + "constraint": true, + "primary": true, + "unique": true + } + ], + "checks": [], + "foreignKeys": {}, + "nativeEnums": {} + }, { "columns": { "bi_job_index": { diff --git a/apps/mis-server/src/migrations/Migration20240507034022.ts b/apps/mis-server/src/migrations/Migration20240507034022.ts new file mode 100644 index 0000000000..9e1cf4924b --- /dev/null +++ b/apps/mis-server/src/migrations/Migration20240507034022.ts @@ -0,0 +1,14 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20240507034022 extends Migration { + + async up(): Promise { + this.addSql('create table `cluster` (`id` int unsigned not null auto_increment primary key, `cluster_id` varchar(255) not null, `activation_status` enum(\'ACTIVATED\', \'DEACTIVATED\') not null default \'ACTIVATED\' comment \'ACTIVATED, DEACTIVATED\', `last_activation_operation` json null, `create_time` DATETIME(6) null, `update_time` DATETIME(6) null) default character set utf8mb4 engine = InnoDB;'); + this.addSql('alter table `cluster` add unique `cluster_cluster_id_unique`(`cluster_id`);'); + } + + async down(): Promise { + this.addSql('drop table if exists `cluster`;'); + } + +} diff --git a/apps/mis-server/src/plugins/clusters.ts b/apps/mis-server/src/plugins/clusters.ts index 0076800c48..61bf3c2030 100644 --- a/apps/mis-server/src/plugins/clusters.ts +++ b/apps/mis-server/src/plugins/clusters.ts @@ -13,12 +13,13 @@ import { ServiceError } from "@ddadaal/tsgrpc-common"; import { Logger, plugin } from "@ddadaal/tsgrpc-server"; import { status } from "@grpc/grpc-js"; -import { getLoginNode } from "@scow/config/build/cluster"; +import { ClusterConfigSchema, getLoginNode } from "@scow/config/build/cluster"; import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter"; +import { scowErrorMetadata } from "@scow/lib-server/build/error"; import { testRootUserSshLogin } from "@scow/lib-ssh"; -import { clusters } from "src/config/clusters"; +import { updateCluster } from "src/bl/clustersUtils"; +import { configClusters } from "src/config/clusters"; import { rootKeyPair } from "src/config/env"; -import { scowErrorMetadata } from "src/utils/error"; type CallOnAllResult = { cluster: string; @@ -27,6 +28,8 @@ type CallOnAllResult = { // Throw ServiceError if failed. type CallOnAll = ( + // clusters for calling to connect to adapter client + clusters: Record, logger: Logger, call: (client: SchedulerAdapterClient) => Promise, ) => Promise>; @@ -50,7 +53,7 @@ export const ADAPTER_CALL_ON_ONE_ERROR = "ADAPTER_CALL_ON_ONE_ERROR"; export const clustersPlugin = plugin(async (f) => { if (process.env.NODE_ENV === "production") { - await Promise.all(Object.values(clusters).map(async ({ displayName, loginNodes }) => { + await Promise.all(Object.values(configClusters).map(async ({ displayName, loginNodes }) => { const loginNode = getLoginNode(loginNodes[0]); const address = loginNode.address; const node = loginNode.name; @@ -65,7 +68,12 @@ export const clustersPlugin = plugin(async (f) => { })); } - const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster, c]) => { + // initial clusters database + const configClusterIds = Object.keys(configClusters); + await updateCluster(f.ext.orm.em.fork(), configClusterIds, f.logger); + + // adapterClient of all config clusters + const adapterClientForClusters = Object.entries(configClusters).reduce((prev, [cluster, c]) => { const client = getSchedulerAdapterClient(c.adapterUrl); prev[cluster] = client; @@ -73,13 +81,25 @@ export const clustersPlugin = plugin(async (f) => { return prev; }, {} as Record); + // adapterClients of activated clusters + const getAdapterClientForActivatedClusters = (clustersParam: Record) => { + return Object.entries(clustersParam).reduce((prev, [cluster, c]) => { + const client = getSchedulerAdapterClient(c.adapterUrl); + prev[cluster] = client; + return prev; + }, {} as Record); + }; + const getAdapterClient = (cluster: string) => { return adapterClientForClusters[cluster]; }; + f.logger.child({ plugin: "cluster" }); + const clustersPlugin = { callOnOne: (async (cluster, logger, call) => { + const client = getAdapterClient(cluster); if (!client) { @@ -116,9 +136,11 @@ export const clustersPlugin = plugin(async (f) => { }), // throws error if failed. - callOnAll: (async (logger, call) => { + callOnAll: (async (clusters, logger, call) => { + + const adapterClientForActivatedClusters = getAdapterClientForActivatedClusters(clusters); - const responses = await Promise.all(Object.entries(adapterClientForClusters) + const responses = await Promise.all(Object.entries(adapterClientForActivatedClusters) .map(async ([cluster, client]) => { return call(client).then((result) => { logger.info("Executing on %s success", cluster); diff --git a/apps/mis-server/src/plugins/fetch.ts b/apps/mis-server/src/plugins/fetch.ts index 69e07293ca..bad2c6507c 100644 --- a/apps/mis-server/src/plugins/fetch.ts +++ b/apps/mis-server/src/plugins/fetch.ts @@ -37,7 +37,7 @@ export const fetchPlugin = plugin(async (f) => { if (fetchIsRunning) return; fetchIsRunning = true; - return fetchJobs(f.ext.orm.em.fork(), logger, f.ext, f.ext).finally(() => { fetchIsRunning = false; }); + return fetchJobs(f.ext.orm.em.fork(), logger, f.ext).finally(() => { fetchIsRunning = false; }); }; const task = cron.schedule( diff --git a/apps/mis-server/src/plugins/price.ts b/apps/mis-server/src/plugins/price.ts index 41766f690d..761c986ddb 100644 --- a/apps/mis-server/src/plugins/price.ts +++ b/apps/mis-server/src/plugins/price.ts @@ -11,12 +11,10 @@ */ import { plugin } from "@ddadaal/tsgrpc-server"; -import { createPriceMap, PriceMap } from "src/bl/PriceMap"; +import { createPriceMap } from "src/bl/PriceMap"; export interface PricePlugin { - price: { - createPriceMap: () => Promise; - } + price: {} } @@ -36,8 +34,6 @@ export const pricePlugin = plugin(async (s) => { logger.info("Platform price items are complete. "); } - s.addExtension("price", { - createPriceMap: () => createPriceMap(s.ext.orm.em.fork(), s.ext.clusters, logger), - }); + s.addExtension("price", {}); }); diff --git a/apps/mis-server/src/plugins/syncBlockStatus.ts b/apps/mis-server/src/plugins/syncBlockStatus.ts index a24293510c..2097a9d6e4 100644 --- a/apps/mis-server/src/plugins/syncBlockStatus.ts +++ b/apps/mis-server/src/plugins/syncBlockStatus.ts @@ -39,7 +39,8 @@ export const syncBlockStatusPlugin = plugin(async (f) => { if (synchronizeIsRunning) return; synchronizeIsRunning = true; - return synchronizeBlockStatus(f.ext.orm.em.fork(), logger, f.ext).finally(() => { synchronizeIsRunning = false; }); + return synchronizeBlockStatus(f.ext.orm.em.fork(), logger, f.ext) + .finally(() => { synchronizeIsRunning = false; }); }; const task = cron.schedule( diff --git a/apps/mis-server/src/services/account.ts b/apps/mis-server/src/services/account.ts index 280ac91a60..e3e47e4b53 100644 --- a/apps/mis-server/src/services/account.ts +++ b/apps/mis-server/src/services/account.ts @@ -20,6 +20,7 @@ import { Decimal, decimalToMoney, moneyToNumber } from "@scow/lib-decimal"; import { account_AccountStateFromJSON, AccountServiceServer, AccountServiceService, BlockAccountResponse_Result } from "@scow/protos/build/server/account"; import { blockAccount, unblockAccount } from "src/bl/block"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { authUrl } from "src/config"; import { Account, AccountState } from "src/entities/Account"; import { AccountWhitelist } from "src/entities/AccountWhitelist"; @@ -47,7 +48,9 @@ export const accountServiceServer = plugin((server) => { }; } + const currentActivatedClusters = await getActivatedClusters(em, logger); const jobs = await server.ext.clusters.callOnAll( + currentActivatedClusters, logger, async (client) => { const fields = [ @@ -71,7 +74,9 @@ export const accountServiceServer = plugin((server) => { const blockThresholdAmount = account.blockThresholdAmount ?? account.tenant.$.defaultAccountBlockThreshold; - const result = await blockAccount(account, server.ext.clusters, logger); + const result = await blockAccount(account, + currentActivatedClusters, + server.ext.clusters, logger); if (result === "AlreadyBlocked") { @@ -125,7 +130,6 @@ export const accountServiceServer = plugin((server) => { code: Status.FAILED_PRECONDITION, message: `Account ${accountName} is unblocked`, }; } - // 将账户从被上级封锁或冻结状态变更为正常 account.state = AccountState.NORMAL; @@ -149,7 +153,8 @@ export const accountServiceServer = plugin((server) => { return [{ executed: true }]; } - await unblockAccount(account, server.ext.clusters, logger); + const currentActivatedClusters = await getActivatedClusters(em, logger); + await unblockAccount(account, currentActivatedClusters, server.ext.clusters, logger); return [{ executed: true }]; @@ -246,8 +251,10 @@ export const accountServiceServer = plugin((server) => { throw e; }; + const currentActivatedClusters = await getActivatedClusters(em, logger); logger.info("Creating account in cluster."); await server.ext.clusters.callOnAll( + currentActivatedClusters, logger, async (client) => { await asyncClientCall(client.account, "createAccount", { @@ -371,7 +378,8 @@ export const accountServiceServer = plugin((server) => { // 如果移入白名单之前账户状态不为冻结，则账户状态变更为正常，账户在集群中为解封状态 } else { account.state = AccountState.NORMAL; - await unblockAccount(account, server.ext.clusters, logger); + const currentActivatedClusters = await getActivatedClusters(em, logger); + await unblockAccount(account, currentActivatedClusters, server.ext.clusters, logger); } await em.persistAndFlush(whitelist); @@ -419,7 +427,8 @@ export const accountServiceServer = plugin((server) => { if (shouldBlockInCluster) { logger.info("Account %s is out of balance and not whitelisted. Block the account.", account.accountName); - await blockAccount(account, server.ext.clusters, logger); + const currentActivatedClusters = await getActivatedClusters(em, logger); + await blockAccount(account, currentActivatedClusters, server.ext.clusters, logger); } await em.flush(); @@ -455,15 +464,17 @@ export const accountServiceServer = plugin((server) => { currentBlockThreshold, ).shouldBlockInCluster; + const currentActivatedClusters = await getActivatedClusters(em, logger); + if (shouldBlockInCluster) { logger.info("Account %s may be out of balance. Block the account.", account.accountName); - await blockAccount(account, server.ext.clusters, logger); + await blockAccount(account, currentActivatedClusters, server.ext.clusters, logger); } if (!shouldBlockInCluster) { logger.info("The balance of Account %s is greater than the block threshold amount. " + "Unblock the account.", account.accountName); - await unblockAccount(account, server.ext.clusters, logger); + await unblockAccount(account, currentActivatedClusters, server.ext.clusters, logger); } await em.persistAndFlush(account); diff --git a/apps/mis-server/src/services/admin.ts b/apps/mis-server/src/services/admin.ts index 562dd8a288..7401747cf3 100644 --- a/apps/mis-server/src/services/admin.ts +++ b/apps/mis-server/src/services/admin.ts @@ -14,12 +14,14 @@ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { plugin } from "@ddadaal/tsgrpc-server"; import { ServiceError } from "@grpc/grpc-js"; import { Status } from "@grpc/grpc-js/build/src/constants"; +import { libCheckActivatedClusters } from "@scow/lib-server/build/misCommon/clustersActivation"; import { AdminServiceServer, AdminServiceService, ClusterAccountInfo, ClusterAccountInfo_ImportStatus, } from "@scow/protos/build/server/admin"; import { updateBlockStatusInSlurm } from "src/bl/block"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { importUsers, ImportUsersData } from "src/bl/importUsers"; import { Account } from "src/entities/Account"; import { StorageQuota } from "src/entities/StorageQuota"; @@ -103,7 +105,13 @@ export const adminServiceServer = plugin((server) => { }; } - const reply = await importUsers(data as ImportUsersData, em, whitelist, server.ext.clusters, logger); + const currentActivatedClusters = await getActivatedClusters(em, logger); + + const reply = await importUsers(data as ImportUsersData, + em, + whitelist, + currentActivatedClusters, + server.ext.clusters, logger); return [reply]; @@ -112,6 +120,9 @@ export const adminServiceServer = plugin((server) => { getClusterUsers: async ({ request, em, logger }) => { const { cluster } = request; + const currentActivatedClusters = await getActivatedClusters(em, logger); + libCheckActivatedClusters({ clusterIds: cluster, activatedClusters: currentActivatedClusters, logger }); + const result = await server.ext.clusters.callOnOne( cluster, logger, @@ -215,7 +226,10 @@ export const adminServiceServer = plugin((server) => { return [{}]; }, - syncBlockStatus: async () => { + syncBlockStatus: async ({ em, logger }) => { + // check whether there is activated cluster in SCOW + // cause syncBlockStatus in plugin will skip the check + await getActivatedClusters(em, logger); const reply = await server.ext.syncBlockStatus.sync(); return [reply]; }, diff --git a/apps/mis-server/src/services/charging.ts b/apps/mis-server/src/services/charging.ts index ddc72ab486..22ddafecbc 100644 --- a/apps/mis-server/src/services/charging.ts +++ b/apps/mis-server/src/services/charging.ts @@ -19,6 +19,7 @@ import { SortOrder } from "@scow/protos/build/common/sort_order"; import { ChargeRecord as ChargeRecordProto, ChargingServiceServer, ChargingServiceService } from "@scow/protos/build/server/charging"; import { charge, pay } from "src/bl/charging"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { misConfig } from "src/config/mis"; import { Account } from "src/entities/Account"; import { ChargeRecord } from "src/entities/ChargeRecord"; @@ -91,6 +92,8 @@ export const chargingServiceServer = plugin((server) => { } + const currentActivatedClusters = await getActivatedClusters(em, logger); + return await pay({ amount: new Decimal(moneyToNumber(amount)), comment, @@ -98,7 +101,7 @@ export const chargingServiceServer = plugin((server) => { type, ipAddress, operatorId, - }, em, logger, server.ext); + }, em, currentActivatedClusters, logger, server.ext); }); return [{ @@ -135,6 +138,8 @@ export const chargingServiceServer = plugin((server) => { } } + const currentActivatedClusters = await getActivatedClusters(em, logger); + return await charge({ amount: new Decimal(moneyToNumber(amount)), comment, @@ -142,7 +147,7 @@ export const chargingServiceServer = plugin((server) => { type, userId, metadata, - }, em, logger, server.ext); + }, em, currentActivatedClusters, logger, server.ext); }); return [{ diff --git a/apps/mis-server/src/services/config.ts b/apps/mis-server/src/services/config.ts index b56ae4ddd5..1add68939a 100644 --- a/apps/mis-server/src/services/config.ts +++ b/apps/mis-server/src/services/config.ts @@ -11,11 +11,19 @@ */ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { ServiceError } from "@ddadaal/tsgrpc-common"; import { plugin } from "@ddadaal/tsgrpc-server"; +import { status } from "@grpc/grpc-js"; +import { getClusterConfigs } from "@scow/config/build/cluster"; +import { convertClusterConfigsToServerProtoType, NO_CLUSTERS } from "@scow/lib-server"; +import { scowErrorMetadata } from "@scow/lib-server/build/error"; import { ConfigServiceServer, ConfigServiceService } from "@scow/protos/build/common/config"; +import { updateCluster } from "src/bl/clustersUtils"; export const configServiceServer = plugin((server) => { server.addService(ConfigServiceService, { + + // do not need check cluster's activation getClusterConfig: async ({ request, logger }) => { const { cluster } = request; @@ -28,5 +36,26 @@ export const configServiceServer = plugin((server) => { return [reply]; }, + + getClusterConfigFiles: async ({ em, logger }) => { + + const clusterConfigs = getClusterConfigs(undefined, logger); + + const clusterConfigsProto = convertClusterConfigsToServerProtoType(clusterConfigs); + + const currentConfigClusterIds = Object.keys(clusterConfigs); + if (currentConfigClusterIds.length === 0) { + throw new ServiceError({ + code: status.INTERNAL, + details: "Unable to find cluster configuration files. Please contact the system administrator.", + metadata: scowErrorMetadata(NO_CLUSTERS), + }); + } + // update the activation status of cluster in db + await updateCluster(em, currentConfigClusterIds, logger); + + return [{ clusterConfigs: clusterConfigsProto }]; + }, + }); }); diff --git a/apps/mis-server/src/services/init.ts b/apps/mis-server/src/services/init.ts index 1df05a2f4d..7ede1b7e19 100644 --- a/apps/mis-server/src/services/init.ts +++ b/apps/mis-server/src/services/init.ts @@ -47,19 +47,20 @@ export const initServiceServer = plugin((server) => { // 需要注意，如果扔出异常，前端会根据异常结果显示不同提示 // 显示两种情况，认证系统中创建失败的原因ALREADY_EXISTS_IN_AUTH=>成功 // 显示两种情况，其他错误=>失败 - const user = await createUserInDatabase(userId, name, email, DEFAULT_TENANT_NAME, server.logger, em) - .catch((e) => { - if (e.code === Status.ALREADY_EXISTS) { - throw { - code: Status.ALREADY_EXISTS, - message:`User with userId ${userId} already exists in scow.`, - details: "EXISTS_IN_SCOW", - }; - } - throw { - code: Status.INTERNAL, - message: `Error creating user with userId ${userId} in database.` }; - }); + const user = + await createUserInDatabase(userId, name, email, DEFAULT_TENANT_NAME, server.logger, em) + .catch((e) => { + if (e.code === Status.ALREADY_EXISTS) { + throw { + code: Status.ALREADY_EXISTS, + message:`User with userId ${userId} already exists in scow.`, + details: "EXISTS_IN_SCOW", + }; + } + throw { + code: Status.INTERNAL, + message: `Error creating user with userId ${userId} in database.` }; + }); user.platformRoles.push(PlatformRole.PLATFORM_ADMIN); user.tenantRoles.push(TenantRole.TENANT_ADMIN); diff --git a/apps/mis-server/src/services/job.ts b/apps/mis-server/src/services/job.ts index c71df1e6cb..ea1027e020 100644 --- a/apps/mis-server/src/services/job.ts +++ b/apps/mis-server/src/services/job.ts @@ -18,6 +18,7 @@ import { FilterQuery, QueryOrder, raw, UniqueConstraintViolationException } from import { Decimal, decimalToMoney, moneyToNumber } from "@scow/lib-decimal"; import { jobInfoToRunningjob } from "@scow/lib-scheduler-adapter"; import { checkTimeZone, convertToDateMessage } from "@scow/lib-server/build/date"; +import { libCheckActivatedClusters } from "@scow/lib-server/build/misCommon/clustersActivation"; import { ChargeRecord } from "@scow/protos/build/server/charging"; import { GetJobsResponse, @@ -25,7 +26,8 @@ import { JobFilter, JobServiceServer, JobServiceService } from "@scow/protos/build/server/job"; import { charge, pay } from "src/bl/charging"; -import { getActiveBillingItems } from "src/bl/PriceMap"; +import { getActivatedClusters } from "src/bl/clustersUtils"; +import { createPriceMap, getActiveBillingItems } from "src/bl/PriceMap"; import { misConfig } from "src/config/mis"; import { Account } from "src/entities/Account"; import { JobInfo as JobInfoEntity } from "src/entities/JobInfo"; @@ -142,6 +144,8 @@ export const jobServiceServer = plugin((server) => { const savedFields = misConfig.jobChargeMetadata?.savedFields; + const currentActivatedClusters = await getActivatedClusters(em, logger); + await Promise.all(jobs.map(async (x) => { logger.info("Change the prices of job %s from %s(tenant), $s(account) -> %s(tenant), %s(account)", x.biJobIndex, x.tenantPrice.toFixed(2), x.accountPrice.toFixed(2), @@ -173,7 +177,7 @@ export const jobServiceServer = plugin((server) => { type, amount: newTenantPrice.minus(x.tenantPrice), metadata: metadataMap, - }, em, logger, server.ext); + }, em, currentActivatedClusters, logger, server.ext); } else if (x.tenantPrice.gt(newTenantPrice)) { await pay({ target: account.tenant.$, @@ -182,7 +186,7 @@ export const jobServiceServer = plugin((server) => { operatorId, type, ipAddress, - }, em, logger, server.ext); + }, em, currentActivatedClusters, logger, server.ext); } x.tenantPrice = newTenantPrice; } @@ -196,7 +200,7 @@ export const jobServiceServer = plugin((server) => { amount: newAccountPrice.minus(x.accountPrice), userId: x.user, metadata: metadataMap, - }, em, logger, server.ext); + }, em, currentActivatedClusters, logger, server.ext); } else if (x.accountPrice.gt(newAccountPrice)) { await pay({ target: account, @@ -205,7 +209,7 @@ export const jobServiceServer = plugin((server) => { operatorId, type, ipAddress, - }, em, logger, server.ext); + }, em, currentActivatedClusters, logger, server.ext); } x.accountPrice = newAccountPrice; } @@ -246,6 +250,9 @@ export const jobServiceServer = plugin((server) => { : tenantName !== undefined ? tenantAccounts : []; + const currentActivatedClusters = await getActivatedClusters(em, logger); + libCheckActivatedClusters({ clusterIds: cluster, activatedClusters: currentActivatedClusters, logger }); + const reply = await server.ext.clusters.callOnOne( cluster, logger, @@ -274,9 +281,12 @@ export const jobServiceServer = plugin((server) => { }, - changeJobTimeLimit: async ({ request, logger }) => { + changeJobTimeLimit: async ({ request, em, logger }) => { const { cluster, limitMinutes, jobId } = request; + const currentActivatedClusters = await getActivatedClusters(em, logger); + libCheckActivatedClusters({ clusterIds: cluster, activatedClusters: currentActivatedClusters, logger }); + await server.ext.clusters.callOnOne( cluster, logger, @@ -291,10 +301,13 @@ export const jobServiceServer = plugin((server) => { return [{}]; }, - queryJobTimeLimit: async ({ request, logger }) => { + queryJobTimeLimit: async ({ request, em, logger }) => { const { cluster, jobId } = request; + const currentActivatedClusters = await getActivatedClusters(em, logger); + libCheckActivatedClusters({ clusterIds: cluster, activatedClusters: currentActivatedClusters, logger }); + const reply = await server.ext.clusters.callOnOne( cluster, logger, @@ -345,10 +358,10 @@ export const jobServiceServer = plugin((server) => { historyItems: activeOnly ? [] : billingItems.filter((x) => !activePrices.includes(x)).map(priceItemToGrpc) }]; }, - getMissingDefaultPriceItems: async () => { + getMissingDefaultPriceItems: async ({ em }) => { // check price map completeness - const priceMap = await server.ext.price.createPriceMap(); + const priceMap = await createPriceMap(em, server.ext.clusters, logger); const missingItems = priceMap.getMissingDefaultPriceItems(); return [{ items: missingItems }]; @@ -490,9 +503,12 @@ export const jobServiceServer = plugin((server) => { return [{ count }]; }, - cancelJob: async ({ request, logger }) => { + cancelJob: async ({ request, em, logger }) => { const { cluster, userId, jobId } = request; + const currentActivatedClusters = await getActivatedClusters(em, logger); + libCheckActivatedClusters({ clusterIds: cluster, activatedClusters: currentActivatedClusters, logger }); + await server.ext.clusters.callOnOne( cluster, logger, diff --git a/apps/mis-server/src/services/jobChargeLimit.ts b/apps/mis-server/src/services/jobChargeLimit.ts index 017519cf74..0bb95a18eb 100644 --- a/apps/mis-server/src/services/jobChargeLimit.ts +++ b/apps/mis-server/src/services/jobChargeLimit.ts @@ -19,6 +19,7 @@ import { moneyToNumber } from "@scow/lib-decimal/build/convertion"; import { JobChargeLimitServiceServer, JobChargeLimitServiceService } from "@scow/protos/build/server/job_charge_limit"; import { unblockUserInAccount } from "src/bl/block"; import { setJobCharge } from "src/bl/charging"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { UserAccount, UserStatus } from "src/entities/UserAccount"; import { getUserStateInfo } from "src/utils/accountUserState"; @@ -63,8 +64,11 @@ export const jobChargeLimitServer = plugin((server) => { userAccount.usedJobCharge, ).shouldBlockInCluster; + + const currentActivatedClusters = await getActivatedClusters(em, logger); + if (!shouldBlockUserInCluster) { - await unblockUserInAccount(userAccount, server.ext, logger); + await unblockUserInAccount(userAccount, currentActivatedClusters, server.ext, logger); userAccount.blockedInCluster = UserStatus.UNBLOCKED; } @@ -104,7 +108,10 @@ export const jobChargeLimitServer = plugin((server) => { }; } - await setJobCharge(userAccount, new Decimal(moneyToNumber(limit)), server.ext, logger); + const currentActivatedClusters = await getActivatedClusters(em, logger); + + await setJobCharge(userAccount, + new Decimal(moneyToNumber(limit)), currentActivatedClusters, server.ext, logger); logger.info("Set %s job charge limit to user %s account %s. Current used %s", userAccount.jobChargeLimit!.toFixed(2), diff --git a/apps/mis-server/src/services/misConfig.ts b/apps/mis-server/src/services/misConfig.ts index f387eedcdb..9a80743d91 100644 --- a/apps/mis-server/src/services/misConfig.ts +++ b/apps/mis-server/src/services/misConfig.ts @@ -12,7 +12,11 @@ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { plugin } from "@ddadaal/tsgrpc-server"; -import { ConfigServiceServer, ConfigServiceService } from "@scow/protos/build/server/config"; +import { ServiceError, status } from "@grpc/grpc-js"; +import { ClusterRuntimeInfo_LastActivationOperation, + ConfigServiceServer, ConfigServiceService } from "@scow/protos/build/server/config"; +import { getActivatedClusters, getClustersRuntimeInfo } from "src/bl/clustersUtils"; +import { Cluster, ClusterActivationStatus } from "src/entities/Cluster"; export const misConfigServiceServer = plugin((server) => { server.addService(ConfigServiceService, { @@ -23,10 +27,12 @@ export const misConfigServiceServer = plugin((server) => { * Use the new API function GetAvailablePartitionsForCluster instead. * @deprecated */ - getAvailablePartitions: async ({ request, logger }) => { + getAvailablePartitions: async ({ request, em, logger }) => { const { accountName, userId } = request; + const currentActivatedClusters = await getActivatedClusters(em, logger).catch(); const reply = await server.ext.clusters.callOnAll( + currentActivatedClusters, logger, async (client) => await asyncClientCall(client.config, "getAvailablePartitions", { accountName, userId, @@ -44,6 +50,7 @@ export const misConfigServiceServer = plugin((server) => { getAvailablePartitionsForCluster: async ({ request, logger }) => { const { cluster, accountName, userId } = request; + // do not need check cluster's activation const reply = await server.ext.clusters.callOnOne( cluster, logger, @@ -54,5 +61,107 @@ export const misConfigServiceServer = plugin((server) => { return [reply]; }, + + getClustersRuntimeInfo: async ({ em, logger }) => { + + const reply = await getClustersRuntimeInfo(em, logger); + + return [{ results: reply }]; + }, + + activateCluster: async ({ request, em, logger }) => { + const { clusterId, operatorId } = request; + + const cluster = await em.findOne(Cluster, { clusterId }); + + if (!cluster) { + throw { + code: status.NOT_FOUND, message: `Cluster（ Cluster ID: ${clusterId}） is not found`, + }; + } + + // check current scheduler adapter connection state + // do not need check cluster's activation + await server.ext.clusters.callOnOne( + clusterId, + logger, + async (client) => await asyncClientCall(client.config, "getClusterConfig", {}), + ).catch((e) => { + logger.info("Cluster Connection Error ( Cluster ID : %s , Details: %s ) .", cluster, e); + throw { + code: status.FAILED_PRECONDITION, + message: `Activate cluster failed, Cluster（ Cluster ID: ${clusterId}） is currently unreachable.`, + }; + }); + + // when the cluster has already been activated + if (cluster.activationStatus === ClusterActivationStatus.ACTIVATED) { + logger.info("Cluster (Cluster ID: %s) has already been activated", + clusterId, + ); + return [{ executed: false }]; + } + + cluster.activationStatus = ClusterActivationStatus.ACTIVATED; + + // save operator userId in lastActivationOperation + const lastActivationOperationMap: ClusterRuntimeInfo_LastActivationOperation = {}; + + lastActivationOperationMap["operatorId"] = operatorId; + cluster.lastActivationOperation = lastActivationOperationMap; + + await em.persistAndFlush(cluster); + + logger.info("Cluster (Cluster ID: %s) is successfully activated by user (User Id: %s)", + clusterId, + operatorId, + ); + + return [{ executed: true }]; + + }, + + deactivateCluster: async ({ request, em, logger }) => { + const { clusterId, operatorId, deactivationComment } = request; + + const cluster = await em.findOne(Cluster, { clusterId }); + + if (!cluster) { + throw { + code: status.NOT_FOUND, message: `Cluster（ Cluster ID: ${clusterId}） is not found`, + }; + } + + if (cluster.activationStatus === ClusterActivationStatus.DEACTIVATED) { + + logger.info("Cluster (Cluster ID: %s) has already been deactivated"); + + return [{ executed: false }]; + } + + cluster.activationStatus = ClusterActivationStatus.DEACTIVATED; + + // save operator userId and deactivation in lastActivationOperation + const lastActivationOperationMap: ClusterRuntimeInfo_LastActivationOperation = {}; + lastActivationOperationMap["operatorId"] = operatorId; + + if (deactivationComment) { + lastActivationOperationMap["deactivationComment"] = deactivationComment; + } + cluster.lastActivationOperation = lastActivationOperationMap; + + + await em.persistAndFlush(cluster); + + logger.info("Cluster (Cluster ID: %s) is successfully deactivated by user (User Id: %s) with comment %s", + clusterId, + operatorId, + deactivationComment, + ); + + return [{ executed: true }]; + + }, + }); }); diff --git a/apps/mis-server/src/services/tenant.ts b/apps/mis-server/src/services/tenant.ts index 55919d6329..661a185ba4 100644 --- a/apps/mis-server/src/services/tenant.ts +++ b/apps/mis-server/src/services/tenant.ts @@ -18,6 +18,7 @@ import { createUser } from "@scow/lib-auth"; import { Decimal, decimalToMoney, moneyToNumber } from "@scow/lib-decimal"; import { TenantServiceServer, TenantServiceService } from "@scow/protos/build/server/tenant"; import { blockAccount, unblockAccount } from "src/bl/block"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { authUrl } from "src/config"; import { Account } from "src/entities/Account"; import { Tenant } from "src/entities/Tenant"; @@ -122,24 +123,25 @@ export const tenantServiceServer = plugin((server) => { }); // 在数据库中创建user - const user = await createUserInDatabase(userId, userName, userEmail, tenantName, logger, em) - .then(async (user) => { - user.tenantRoles = [TenantRole.TENANT_ADMIN]; - await em.persistAndFlush(user); - return user; - }).catch((e) => { - if (e.code === Status.ALREADY_EXISTS) { - throw { - code: Status.ALREADY_EXISTS, - message: `User with userId ${userId} already exists in scow.`, - details: "USER_ALREADY_EXISTS", - }; - } - throw { - code: Status.INTERNAL, - message: `Error creating user with userId ${userId} in database.`, - }; - }); + const user = + await createUserInDatabase(userId, userName, userEmail, tenantName, logger, em) + .then(async (user) => { + user.tenantRoles = [TenantRole.TENANT_ADMIN]; + await em.persistAndFlush(user); + return user; + }).catch((e) => { + if (e.code === Status.ALREADY_EXISTS) { + throw { + code: Status.ALREADY_EXISTS, + message: `User with userId ${userId} already exists in scow.`, + details: "USER_ALREADY_EXISTS", + }; + } + throw { + code: Status.INTERNAL, + message: `Error creating user with userId ${userId} in database.`, + }; + }); // call auth const createdInAuth = await createUser(authUrl, { identityId: user.userId, id: user.id, mail: user.email, name: user.name, password: userPassword }, @@ -181,6 +183,8 @@ export const tenantServiceServer = plugin((server) => { const accounts = await em.find(Account, { tenant: tenant, blockThresholdAmount : {} }, { populate: ["tenant"], }); + + const currentActivatedClusters = await getActivatedClusters(em, logger); if (accounts.length > 0) { await Promise.allSettled(accounts .map(async (account) => { @@ -195,13 +199,13 @@ export const tenantServiceServer = plugin((server) => { if (shouldBlockInCluster) { logger.info("Account %s may be out of balance when using default tenant block threshold amount. " + "Block the account.", account.accountName); - await blockAccount(account, server.ext.clusters, logger); + await blockAccount(account, currentActivatedClusters, server.ext.clusters, logger); } if (!shouldBlockInCluster) { logger.info("The balance of Account %s is greater than the default tenant block threshold amount. " + "Unblock the account.", account.accountName); - await unblockAccount(account, server.ext.clusters, logger); + await unblockAccount(account, currentActivatedClusters, server.ext.clusters, logger); } }), ).catch((e) => { diff --git a/apps/mis-server/src/services/user.ts b/apps/mis-server/src/services/user.ts index 99df31406f..ba6835ec68 100644 --- a/apps/mis-server/src/services/user.ts +++ b/apps/mis-server/src/services/user.ts @@ -32,8 +32,8 @@ import { UserServiceService, UserStatus as PFUserStatus } from "@scow/protos/build/server/user"; import { blockUserInAccount, unblockUserInAccount } from "src/bl/block"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { authUrl } from "src/config"; -import { clusters } from "src/config/clusters"; import { Account } from "src/entities/Account"; import { Tenant } from "src/entities/Tenant"; import { PlatformRole, TenantRole, User } from "src/entities/User"; @@ -188,12 +188,15 @@ export const userServiceServer = plugin((server) => { }; } - await server.ext.clusters.callOnAll(logger, async (client) => { + const currentActivatedClusters = await getActivatedClusters(em, logger); + + await server.ext.clusters.callOnAll(currentActivatedClusters, logger, async (client) => { return await asyncClientCall(client.user, "addUserToAccount", { userId, accountName }); }).catch(async (e) => { // 如果每个适配器返回的Error都是ALREADY_EXISTS，说明所有集群均已添加成功，可以在scow数据库及认证系统中加入该条关系， // 除此以外，都抛出异常 - if (countSubstringOccurrences(e.details, "Error: 6 ALREADY_EXISTS") !== Object.keys(clusters).length) { + if (countSubstringOccurrences(e.details, "Error: 6 ALREADY_EXISTS") + !== Object.keys(currentActivatedClusters).length) { throw e; } }); @@ -237,15 +240,17 @@ export const userServiceServer = plugin((server) => { }; } + const currentActivatedClusters = await getActivatedClusters(em, logger); // 如果要从账户中移出用户，先封锁，先将用户封锁，保证用户无法提交作业 if (userAccount.blockedInCluster === UserStatus.UNBLOCKED) { userAccount.state = UserStateInAccount.BLOCKED_BY_ADMIN; - await blockUserInAccount(userAccount, server.ext, logger); + await blockUserInAccount(userAccount, currentActivatedClusters, server.ext, logger); await em.flush(); } // 查询用户是否有RUNNING、PENDING的作业，如果有，抛出异常 const jobs = await server.ext.clusters.callOnAll( + currentActivatedClusters, logger, async (client) => { const fields = ["job_id", "user", "state", "account"]; @@ -265,12 +270,14 @@ export const userServiceServer = plugin((server) => { }; } - await server.ext.clusters.callOnAll(logger, async (client) => { + + await server.ext.clusters.callOnAll(currentActivatedClusters, logger, async (client) => { return await asyncClientCall(client.user, "removeUserFromAccount", { userId, accountName }); }).catch(async (e) => { // 如果每个适配器返回的Error都是NOT_FOUND，说明所有集群均已将此用户移出账户，可以在scow数据库及认证系统中删除该条关系， // 除此以外，都抛出异常 - if (countSubstringOccurrences(e.details, "Error: 5 NOT_FOUND") !== Object.keys(clusters).length) { + if (countSubstringOccurrences(e.details, "Error: 5 NOT_FOUND") + !== Object.keys(currentActivatedClusters).length) { throw e; } }); @@ -306,7 +313,8 @@ export const userServiceServer = plugin((server) => { }; } - await blockUserInAccount(user, server.ext, logger); + const currentActivatedClusters = await getActivatedClusters(em, logger); + await blockUserInAccount(user, currentActivatedClusters, server.ext, logger); user.state = UserStateInAccount.BLOCKED_BY_ADMIN; user.blockedInCluster = UserStatus.BLOCKED; @@ -342,8 +350,9 @@ export const userServiceServer = plugin((server) => { user.usedJobCharge, ).shouldBlockInCluster; + const currentActivatedClusters = await getActivatedClusters(em, logger); if (!stillBlockUserInCluster) { - await unblockUserInAccount(user, server.ext, logger); + await unblockUserInAccount(user, currentActivatedClusters, server.ext, logger); user.blockedInCluster = UserStatus.UNBLOCKED; } user.state = UserStateInAccount.NORMAL; @@ -411,7 +420,8 @@ export const userServiceServer = plugin((server) => { */ createUser: async ({ request, em, logger }) => { const { name, tenantName, email, identityId, password } = request; - const user = await createUserInDatabase(identityId, name, email, tenantName, server.logger, em) + const user = + await createUserInDatabase(identityId, name, email, tenantName, server.logger, em) .catch((e) => { if (e.code === Status.ALREADY_EXISTS) { throw { @@ -463,19 +473,20 @@ export const userServiceServer = plugin((server) => { */ addUser: async ({ request, em, logger }) => { const { name, tenantName, email, identityId } = request; - const user = await createUserInDatabase(identityId, name, email, tenantName, server.logger, em) - .catch((e) => { - if (e.code === Status.ALREADY_EXISTS) { - throw { - code: Status.ALREADY_EXISTS, - message: `User with userId ${identityId} already exists in scow.`, - details: "EXISTS_IN_SCOW", - }; - } - throw { - code: Status.INTERNAL, - message: `Error creating user with userId ${identityId} in database.` }; - }); + const user + = await createUserInDatabase(identityId, name, email, tenantName, server.logger, em) + .catch((e) => { + if (e.code === Status.ALREADY_EXISTS) { + throw { + code: Status.ALREADY_EXISTS, + message: `User with userId ${identityId} already exists in scow.`, + details: "EXISTS_IN_SCOW", + }; + } + throw { + code: Status.INTERNAL, + message: `Error creating user with userId ${identityId} in database.` }; + }); await callHook("userAdded", { tenantName, userId: user.userId }, logger); diff --git a/apps/mis-server/src/tasks/fetch.ts b/apps/mis-server/src/tasks/fetch.ts index 9907144fbe..335a6ed0b0 100644 --- a/apps/mis-server/src/tasks/fetch.ts +++ b/apps/mis-server/src/tasks/fetch.ts @@ -13,20 +13,20 @@ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { Logger } from "@ddadaal/tsgrpc-server"; import { QueryOrder } from "@mikro-orm/core"; -import { SqlEntityManager } from "@mikro-orm/mysql"; +import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; import { parsePlaceholder } from "@scow/lib-config"; import { ChargeRecord } from "@scow/protos/build/server/charging"; import { GetJobsResponse, JobInfo as ClusterJobInfo } from "@scow/scheduler-adapter-protos/build/protos/job"; import { addJobCharge, charge } from "src/bl/charging"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { emptyJobPriceInfo } from "src/bl/jobPrice"; -import { clusters } from "src/config/clusters"; +import { createPriceMap } from "src/bl/PriceMap"; import { misConfig } from "src/config/mis"; import { Account } from "src/entities/Account"; import { JobInfo } from "src/entities/JobInfo"; import { UserAccount } from "src/entities/UserAccount"; import { ClusterPlugin } from "src/plugins/clusters"; import { callHook } from "src/plugins/hookClient"; -import { PricePlugin } from "src/plugins/price"; import { toGrpc } from "src/utils/job"; async function getClusterLatestDate(em: SqlEntityManager, cluster: string, logger: Logger) { @@ -63,10 +63,9 @@ const processGetJobsResult = (cluster: string, result: GetJobsResponse) => { export let lastFetched: Date | null = null; export async function fetchJobs( - em: SqlEntityManager, + em: SqlEntityManager, logger: Logger, clusterPlugin: ClusterPlugin, - pricePlugin: PricePlugin, ) { logger.info("Start fetching."); @@ -76,10 +75,13 @@ export async function fetchJobs( const accountTenantMap = new Map(accounts.map((x) => [x.accountName, x.tenant.$.name])); - const priceMap = await pricePlugin.price.createPriceMap(); + const priceMap = await createPriceMap(em, clusterPlugin["clusters"], logger); const persistJobAndCharge = async (jobs: ({ cluster: string } & ClusterJobInfo)[]) => { const result = await em.transactional(async (em) => { + + const currentActivatedClusters = await getActivatedClusters(em, logger); + // Calculate prices for new info and persist const pricedJobs: JobInfo[] = []; let pricedJob: JobInfo; @@ -145,7 +147,7 @@ export async function fetchJobs( target: account, userId: pricedJob.user, metadata: metadataMap, - }, em, logger, clusterPlugin); + }, em, currentActivatedClusters, logger, clusterPlugin); // charge tenant await charge({ @@ -155,7 +157,7 @@ export async function fetchJobs( target: account.tenant.$, userId: pricedJob.user, metadata: metadataMap, - }, em, logger, clusterPlugin); + }, em, currentActivatedClusters, logger, clusterPlugin); const ua = await em.findOne(UserAccount, { account: { accountName: pricedJob.account }, @@ -169,7 +171,7 @@ export async function fetchJobs( "User %s in account %s is not found.", pricedJob.user, pricedJob.account); } else { // 用户限额及相关操作 - await addJobCharge(ua, pricedJob.accountPrice, clusterPlugin, logger); + await addJobCharge(ua, pricedJob.accountPrice, currentActivatedClusters, clusterPlugin, logger); } } @@ -188,9 +190,12 @@ export async function fetchJobs( return result.length; }; + const clusters = await getActivatedClusters(em, logger); + try { let newJobsCount = 0; for (const cluster of Object.keys(clusters)) { + logger.info(`fetch jobs from cluster ${cluster}`); const latestDate = await getClusterLatestDate(em, cluster, logger); diff --git a/apps/mis-server/src/utils/createUser.ts b/apps/mis-server/src/utils/createUser.ts index 98105768ab..76369d411a 100644 --- a/apps/mis-server/src/utils/createUser.ts +++ b/apps/mis-server/src/utils/createUser.ts @@ -17,25 +17,31 @@ import { UniqueConstraintViolationException } from "@mikro-orm/core"; import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; import { getLoginNode } from "@scow/config/build/cluster"; import { insertKeyAsUser } from "@scow/lib-ssh"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { rootKeyPair } from "src/config/env"; import { StorageQuota } from "src/entities/StorageQuota"; import { Tenant } from "src/entities/Tenant"; import { User } from "src/entities/User"; export async function createUserInDatabase( - userId: string, name: string, email: string, tenantName: string, logger: Logger, em: SqlEntityManager) { + userId: string, + name: string, + email: string, + tenantName: string, + logger: Logger, + em: SqlEntityManager) { // get default tenant const tenant = await em.findOne(Tenant, { name: tenantName }); if (!tenant) { throw { code: Status.NOT_FOUND, message: `Tenant ${tenantName} is not found.` }; } + // new the user const user = new User({ email, name, tenant, userId, }); - user.storageQuotas.add(Object.keys(clusters).map((x) => new StorageQuota({ + user.storageQuotas.add(Object.keys(configClusters).map((x) => new StorageQuota({ cluster: x, storageQuota: 0, user: user!, @@ -56,10 +62,15 @@ export async function createUserInDatabase( return user; } -export async function insertKeyToNewUser(userId: string, password: string, logger: Logger) { +export async function insertKeyToNewUser( + userId: string, + password: string, + logger: Logger, +) { // Making an ssh Request to the login node as the user created. if (process.env.NODE_ENV === "production") { - await Promise.all(Object.values(clusters).map(async ({ displayName, loginNodes }) => { + + await Promise.all(Object.values(configClusters).map(async ({ displayName, loginNodes }) => { const node = getLoginNode(loginNodes[0]); logger.info("Checking if user can login to %s by login node %s", displayName, node.name); diff --git a/apps/mis-server/tests/admin/clusterActivation.test.ts b/apps/mis-server/tests/admin/clusterActivation.test.ts new file mode 100644 index 0000000000..2e131c52ff --- /dev/null +++ b/apps/mis-server/tests/admin/clusterActivation.test.ts @@ -0,0 +1,432 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { Server } from "@ddadaal/tsgrpc-server"; +import { ChannelCredentials } from "@grpc/grpc-js"; +import { Status } from "@grpc/grpc-js/build/src/constants"; +import { moneyToNumber, numberToMoney } from "@scow/lib-decimal"; +import { AccountServiceClient } from "@scow/protos/build/server/account"; +import { AdminServiceClient } from "@scow/protos/build/server/admin"; +import { ChargingServiceClient } from "@scow/protos/build/server/charging"; +import { ClusterActivationStatus as ClusterActivationStatusProto, + ConfigServiceClient } from "@scow/protos/build/server/config"; +import { createServer } from "src/app"; +import { Account, AccountState } from "src/entities/Account"; +import { Cluster, ClusterActivationStatus } from "src/entities/Cluster"; +import { reloadEntity } from "src/utils/orm"; +import { InitialData, insertInitialData } from "tests/data/data"; +import { dropDatabase } from "tests/data/helpers"; + +let server: Server; +let client: ConfigServiceClient; +let clusterItem: Cluster; +let data: InitialData; + +beforeEach(async () => { + server = await createServer(); + data = await insertInitialData(server.ext.orm.em.fork()); + await server.start(); + + clusterItem = new Cluster({ + clusterId: "hpcTest", + activationStatus: ClusterActivationStatus.DEACTIVATED, + lastActivationOperation: { "operatorId": "userA", "deactivationComment": "Deactivation Comment" }, + }); + const hpc00 = await server.ext.orm.em.fork().findOneOrFail(Cluster, { + clusterId: "hpc00", + }); + hpc00.activationStatus = ClusterActivationStatus.DEACTIVATED; + hpc00.lastActivationOperation = { + "operatorId": "userB", + "deactivationComment": "new deactivation message for upgrade", + }; + + await server.ext.orm.em.fork().persistAndFlush([clusterItem, hpc00]); + + client = new ConfigServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + +}); + +afterEach(async () => { + await dropDatabase(server.ext.orm); + await server.close(); +}); + +it("gets clusters initial database info", async () => { + + const clustersRuntimeInfo = await asyncClientCall(client, "getClustersRuntimeInfo", {}); + + expect(clustersRuntimeInfo.results.length).toEqual(4); + expect(clustersRuntimeInfo.results.map((x) => ({ + clusterId: x.clusterId, + activationStatus: x.activationStatus, + lastActivationOperation: x.lastActivationOperation, + }))).toIncludeSameMembers([ + { + clusterId: "hpc00", + activationStatus: ClusterActivationStatusProto.DEACTIVATED, + lastActivationOperation: { + "operatorId": "userB", + "deactivationComment": "new deactivation message for upgrade", + }, + }, + { + clusterId: "hpc01", + activationStatus: ClusterActivationStatusProto.ACTIVATED, + lastActivationOperation: undefined, + }, + { + clusterId: "hpc02", + activationStatus: ClusterActivationStatusProto.ACTIVATED, + lastActivationOperation: undefined, + }, + { + clusterId: "hpcTest", + activationStatus: ClusterActivationStatusProto.DEACTIVATED, + lastActivationOperation: { "operatorId": "userA", "deactivationComment": "Deactivation Comment" }, + }, + ]); + +}); + +it("cannot activate a cluster if the schedular adapter is not reachable", async () => { + + const reply = await asyncClientCall(client, "activateCluster", { + clusterId: "hpcTest", + operatorId: "userB", + }).catch((e) => e); + expect(reply.code).toBe(Status.FAILED_PRECONDITION); + +}); + +it("cannot write to db when activated a cluster has already been activated", async () => { + + const reply = await asyncClientCall(client, "activateCluster", { + clusterId: "hpc01", + operatorId: "userB", + }); + + expect(reply.executed).toBeFalse(); + + const activatedCluster = await server.ext.orm.em.fork().findOneOrFail(Cluster, { + clusterId: "hpc01", + }); + + expect(activatedCluster.lastActivationOperation).toBeUndefined; +}); + +it("activates a cluster", async () => { + + const reply = await asyncClientCall(client, "activateCluster", { + clusterId: "hpc00", + operatorId: "userC", + }); + expect(reply.executed).toBeTrue(); + + const updatedCluster = await server.ext.orm.em.fork().findOneOrFail(Cluster, { + clusterId: "hpc00", + }); + expect(updatedCluster.activationStatus).toBe(ClusterActivationStatus.ACTIVATED); + expect(updatedCluster.lastActivationOperation).toStrictEqual({ + "operatorId": "userC", + }); + +}); + + +it("cannot deactivate a cluster if not found", async () => { + + const reply = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc123", + operatorId: "userA", + deactivationComment: "deactivation for upgrade", + }).catch((e) => e); + expect(reply.code).toBe(Status.NOT_FOUND); + +}); + +it("cannot write to db when deactivated a cluster has already been deactivated", async () => { + + const reply = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpcTest", + operatorId: "userB", + deactivationComment: "deactivation for upgrade", + }); + + expect(reply.executed).toBeFalse(); + + const deactivatedCluster = await server.ext.orm.em.fork().findOneOrFail(Cluster, { + clusterId: "hpcTest", + }); + expect(deactivatedCluster.activationStatus).toBe(ClusterActivationStatus.DEACTIVATED); + expect(deactivatedCluster.lastActivationOperation).toStrictEqual({ + "operatorId": "userA", + "deactivationComment": "Deactivation Comment", + }); +}); + +it("deactivates a cluster", async () => { + + const reply = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc01", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(reply.executed).toBeTrue(); + + const deactivatedCluster = await server.ext.orm.em.fork().findOneOrFail(Cluster, { + clusterId: "hpc01", + }); + expect(deactivatedCluster.activationStatus).toBe(ClusterActivationStatus.DEACTIVATED); + expect(deactivatedCluster.lastActivationOperation).toStrictEqual({ + "operatorId": "userB", + "deactivationComment": "deactivation message for upgrade", + }); + +}); + +it("creates an account and executes pay operation successfully during cluster activation operation", async () => { + + const reply = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc01", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(reply.executed).toBeTrue(); + + const accountClient = new AccountServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + await asyncClientCall(accountClient, "createAccount", { accountName: "a1234", tenantName: data.tenant.name, + ownerId: data.userA.userId }); + const em = server.ext.orm.em.fork(); + + const account = await em.findOneOrFail(Account, { accountName: "a1234" }); + expect(account.accountName).toBe("a1234"); + expect(account.balance.toNumber()).toBe(0); + expect(account.state).toBe(AccountState.NORMAL); + expect(account.blockedInCluster).toBe(true); + + const amount = numberToMoney(10); + + const chargeClient = new ChargingServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + const payReply = await asyncClientCall(chargeClient, "pay", { + tenantName: data.tenant.name, + accountName: "a1234", + amount: amount, + comment: "comment", + operatorId: "tester", + ipAddress: "127.0.0.1", + type: "test", + }); + + expect(moneyToNumber(payReply.previousBalance!)).toBe(0); + expect(moneyToNumber(payReply.currentBalance!)).toBe(10); + + await reloadEntity(em, account); + + expect(account.balance.toNumber()).toBe(10); + expect(account.blockedInCluster).toBeFalse(); +}); + +it("cannot execute pay operation during all clusters were deactivated", async () => { + + // create account + const accountClient = new AccountServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + await asyncClientCall(accountClient, "createAccount", { accountName: "a1234", tenantName: data.tenant.name, + ownerId: data.userA.userId }); + const em = server.ext.orm.em.fork(); + + const account = await em.findOneOrFail(Account, { accountName: "a1234" }); + expect(account.accountName).toBe("a1234"); + expect(account.balance.toNumber()).toBe(0); + expect(account.state).toBe(AccountState.NORMAL); + expect(account.blockedInCluster).toBeTrue(); + + // deactivate all clusters + const deactivationReply1 = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc01", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(deactivationReply1.executed).toBeTrue(); + + const deactivationReply2 = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc02", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(deactivationReply2.executed).toBeTrue(); + + + // pay operation + const amount = numberToMoney(10); + + const chargeClient = new ChargingServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + const payReply = await asyncClientCall(chargeClient, "pay", { + tenantName: data.tenant.name, + accountName: "a1234", + amount: amount, + comment: "comment", + operatorId: "tester", + ipAddress: "127.0.0.1", + type: "test", + }).catch((e) => e); + + expect(payReply.code).toBe(Status.INTERNAL); + expect(payReply.details).toBe("No available clusters. Please try again later"); + +}); + +it("creates an account and executes charge operation successfully during cluster activation operation", async () => { + // create an account + const accountClient = new AccountServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + await asyncClientCall(accountClient, "createAccount", { accountName: "a1234", tenantName: data.tenant.name, + ownerId: data.userA.userId }); + const em = server.ext.orm.em.fork(); + + const account = await em.findOneOrFail(Account, { accountName: "a1234" }); + expect(account.accountName).toBe("a1234"); + expect(account.balance.toNumber()).toBe(0); + expect(account.state).toBe(AccountState.NORMAL); + expect(account.blockedInCluster).toBe(true); + + // deactivate a cluster + const reply = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc01", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(reply.executed).toBe(true); + + // charge + const amount = numberToMoney(10); + + const chargeClient = new ChargingServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + const chargeReply = await asyncClientCall(chargeClient, "charge", { + tenantName: data.tenant.name, + accountName: "a1234", + type: "123", + amount: amount, + comment: "comment", + }); + + expect(moneyToNumber(chargeReply.previousBalance!)).toBe(0); + expect(moneyToNumber(chargeReply.currentBalance!)).toBe(-10); + + await reloadEntity(em, account); + + expect(account.balance.toNumber()).toBe(-10); + expect(account.blockedInCluster).toBeTruthy(); + expect(account.state).toBe(AccountState.NORMAL); + expect(account.blockedInCluster).toBeTrue(); +}); + +it("cannot execute charge operation during all clusters were deactivated", async () => { + + // create account + const accountClient = new AccountServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + await asyncClientCall(accountClient, "createAccount", { accountName: "a1234", tenantName: data.tenant.name, + ownerId: data.userA.userId }); + const em = server.ext.orm.em.fork(); + + const account = await em.findOneOrFail(Account, { accountName: "a1234" }); + expect(account.accountName).toBe("a1234"); + expect(account.balance.toNumber()).toBe(0); + expect(account.state).toBe(AccountState.NORMAL); + expect(account.blockedInCluster).toBeTrue(); + + // deactivate all clusters + const deactivationReply1 = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc01", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(deactivationReply1.executed).toBeTrue(); + + const deactivationReply2 = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc02", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(deactivationReply2.executed).toBeTrue(); + + + // pay operation + const amount = numberToMoney(10); + + const chargeClient = new ChargingServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + + const chargeReply = await asyncClientCall(chargeClient, "charge", { + tenantName: data.tenant.name, + accountName: "a1234", + type: "123", + amount: amount, + comment: "comment", + }).catch((e) => e); + + expect(chargeReply.code).toBe(Status.INTERNAL); + expect(chargeReply.details).toBe("No available clusters. Please try again later"); + +}); + + +it("cannot import users and accounts during all clusters were deactivated", async () => { + + const data = { + accounts: [ + { + accountName: "a_user1", + users: [{ userId: "user1", userName: "user1Name", blocked: false }, + { userId: "user2", userName: "user2", blocked: true }], + owner: "user1", + blocked: false, + }, + { + accountName: "account2", + users: [{ userId: "user2", userName: "user2", blocked: false }, + { userId: "user3", userName: "user3", blocked: true }], + owner: "user2", + blocked: false, + }, + ], + }; + + // deactivate all clusters + const deactivationReply1 = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc01", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(deactivationReply1.executed).toBeTrue(); + + const deactivationReply2 = await asyncClientCall(client, "deactivateCluster", { + clusterId: "hpc02", + operatorId: "userB", + deactivationComment: "deactivation message for upgrade", + }); + expect(deactivationReply2.executed).toBeTrue(); + + const adminClient = new AdminServiceClient(server.serverAddress, ChannelCredentials.createInsecure()); + const importReply = await asyncClientCall(adminClient, "importUsers", { data: data, whitelist: true }) + .catch((e) => e); + + expect(importReply.code).toBe(Status.INTERNAL); + expect(importReply.details).toBe("No available clusters. Please try again later"); +}); + diff --git a/apps/mis-server/tests/admin/jobChargeLimit.test.ts b/apps/mis-server/tests/admin/jobChargeLimit.test.ts index 2f658f85fb..a64475ee1b 100644 --- a/apps/mis-server/tests/admin/jobChargeLimit.test.ts +++ b/apps/mis-server/tests/admin/jobChargeLimit.test.ts @@ -15,18 +15,19 @@ import { Server } from "@ddadaal/tsgrpc-server"; import { ChannelCredentials } from "@grpc/grpc-js"; import { Status } from "@grpc/grpc-js/build/src/constants"; import { Loaded } from "@mikro-orm/core"; -import { SqlEntityManager } from "@mikro-orm/mysql"; +import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; import { Decimal, decimalToMoney } from "@scow/lib-decimal"; import { JobChargeLimitServiceClient } from "@scow/protos/build/server/job_charge_limit"; import { createServer } from "src/app"; import { addJobCharge } from "src/bl/charging"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { UserAccount, UserStateInAccount, UserStatus } from "src/entities/UserAccount"; import { reloadEntity } from "src/utils/orm"; import { InitialData, insertInitialData } from "tests/data/data"; import { dropDatabase } from "tests/data/helpers"; let server: Server; -let em: SqlEntityManager; +let em: SqlEntityManager; let client: JobChargeLimitServiceClient; let data: InitialData; let ua: Loaded; @@ -188,7 +189,9 @@ it("adds job charge", async () => { const charge = new Decimal(20.4); - await addJobCharge(ua, charge, server.ext, server.logger); + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + + await addJobCharge(ua, charge, currentActivatedClusters, server.ext, server.logger); expectDecimalEqual(ua.usedJobCharge, charge); expectDecimalEqual(ua.jobChargeLimit, limit); @@ -204,7 +207,10 @@ it("blocks user if used > limit", async () => { expectDecimalEqual(ua.jobChargeLimit, limit); const charge = new Decimal(120.4); - await addJobCharge(ua, charge, server.ext, server.logger); + + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + + await addJobCharge(ua, charge, currentActivatedClusters, server.ext, server.logger); expectDecimalEqual(ua.usedJobCharge, charge); expectDecimalEqual(ua.jobChargeLimit, limit); @@ -220,7 +226,10 @@ it("blocks user if used = limit", async () => { expectDecimalEqual(ua.jobChargeLimit, limit); const charge = new Decimal(100); - await addJobCharge(ua, charge, server.ext, server.logger); + + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + + await addJobCharge(ua, charge, currentActivatedClusters, server.ext, server.logger); expectDecimalEqual(ua.usedJobCharge, charge); expectDecimalEqual(ua.jobChargeLimit, limit); @@ -254,7 +263,9 @@ it("unblocks user if limit > used is positive and state is normal", async () => const charge = new Decimal(-20.4); - await addJobCharge(ua, charge, server.ext, server.logger); + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + + await addJobCharge(ua, charge, currentActivatedClusters, server.ext, server.logger); expectDecimalEqual(ua.jobChargeLimit, limit); expectDecimalEqual(ua.usedJobCharge, new Decimal(99.6)); @@ -272,7 +283,9 @@ it("still block user if limit > used is positive and state is BLOCKED_BY_ADMIN", const charge = new Decimal(-20.4); - await addJobCharge(ua, charge, server.ext, server.logger); + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + + await addJobCharge(ua, charge, currentActivatedClusters, server.ext, server.logger); expectDecimalEqual(ua.jobChargeLimit, limit); expectDecimalEqual(ua.usedJobCharge, new Decimal(99.6)); @@ -281,8 +294,13 @@ it("still block user if limit > used is positive and state is BLOCKED_BY_ADMIN", }); -it("does nothing if no limit", async () => { const charge = new Decimal(120.4); - await addJobCharge(ua, charge, server.ext, server.logger); +it("does nothing if no limit", async () => { + + const charge = new Decimal(120.4); + + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + + await addJobCharge(ua, charge, currentActivatedClusters, server.ext, server.logger); expect(ua.jobChargeLimit).toBeUndefined(); expect(ua.usedJobCharge).toBeUndefined(); diff --git a/apps/mis-server/tests/admin/whitelist.test.ts b/apps/mis-server/tests/admin/whitelist.test.ts index 9d528797d2..125d6e6909 100644 --- a/apps/mis-server/tests/admin/whitelist.test.ts +++ b/apps/mis-server/tests/admin/whitelist.test.ts @@ -14,11 +14,12 @@ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { Server } from "@ddadaal/tsgrpc-server"; import { ChannelCredentials } from "@grpc/grpc-js"; import { Loaded } from "@mikro-orm/core"; -import { SqlEntityManager } from "@mikro-orm/mysql"; +import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; import { Decimal, decimalToMoney } from "@scow/lib-decimal"; import { AccountServiceClient } from "@scow/protos/build/server/account"; import { createServer } from "src/app"; import { charge } from "src/bl/charging"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { Account } from "src/entities/Account"; import { AccountWhitelist } from "src/entities/AccountWhitelist"; import { reloadEntity, toRef } from "src/utils/orm"; @@ -26,7 +27,7 @@ import { InitialData, insertInitialData } from "tests/data/data"; import { dropDatabase } from "tests/data/helpers"; let server: Server; -let em: SqlEntityManager; +let em: SqlEntityManager; let client: AccountServiceClient; let data: InitialData; let a: Loaded; @@ -142,12 +143,14 @@ it("charges user but don't block account if account is whitelist", async () => { await em.flush(); + const currentActivatedClusters = await getActivatedClusters(em, server.logger); + const { currentBalance, previousBalance } = await charge({ amount: new Decimal(2), comment: "", target: a, type: "haha", - }, em.fork(), server.logger, server.ext); + }, em.fork(), currentActivatedClusters, server.logger, server.ext); await reloadEntity(em, a); diff --git a/apps/mis-server/tests/data/data.ts b/apps/mis-server/tests/data/data.ts index 30b5eb998a..fdbc57c9af 100644 --- a/apps/mis-server/tests/data/data.ts +++ b/apps/mis-server/tests/data/data.ts @@ -136,4 +136,3 @@ export async function insertBlockedData(em: SqlEntityManager) { } export type BlockedData = Awaited>; - diff --git a/apps/mis-server/tests/init/init.test.ts b/apps/mis-server/tests/init/init.test.ts index ac47018810..ab195c6c0e 100644 --- a/apps/mis-server/tests/init/init.test.ts +++ b/apps/mis-server/tests/init/init.test.ts @@ -26,7 +26,6 @@ import { dropDatabase } from "tests/data/helpers"; let server: Server; let client: InitServiceClient; - beforeEach(async () => { server = await createServer(); await server.start(); diff --git a/apps/mis-server/tests/job/fetchJobs.test.ts b/apps/mis-server/tests/job/fetchJobs.test.ts index ade57d7571..9eef853291 100644 --- a/apps/mis-server/tests/job/fetchJobs.test.ts +++ b/apps/mis-server/tests/job/fetchJobs.test.ts @@ -16,6 +16,7 @@ import { MySqlDriver, SqlEntityManager } from "@mikro-orm/mysql"; import { Decimal } from "@scow/lib-decimal"; import { createServer } from "src/app"; import { setJobCharge } from "src/bl/charging"; +import { getActivatedClusters } from "src/bl/clustersUtils"; import { emptyJobPriceInfo } from "src/bl/jobPrice"; import { JobInfo } from "src/entities/JobInfo"; import { UserStatus } from "src/entities/UserAccount"; @@ -53,11 +54,12 @@ afterEach(async () => { it("fetches the data", async () => { // set job charge limit of user b in account b + const currentActivatedClusters = await getActivatedClusters(initialEm, server.logger); - await setJobCharge(data.uaBB, new Decimal(0.01), server.ext, server.logger); + await setJobCharge(data.uaBB, new Decimal(0.01), currentActivatedClusters, server.ext, server.logger); await initialEm.flush(); - await fetchJobs(server.ext.orm.em.fork(), server.logger, server.ext, server.ext); + await fetchJobs(server.ext.orm.em.fork(), server.logger, server.ext); const em = server.ext.orm.em.fork(); @@ -137,7 +139,7 @@ it("jobs can be imported when jobs from other clusters already exist in the data await em.persistAndFlush(existedJob); - await fetchJobs(server.ext.orm.em.fork(), server.logger, server.ext, server.ext); + await fetchJobs(server.ext.orm.em.fork(), server.logger, server.ext); const jobs = await em.find(JobInfo, {}); diff --git a/apps/mis-web/config.js b/apps/mis-web/config.js index 3462f89dac..9e4ae1dd8e 100644 --- a/apps/mis-web/config.js +++ b/apps/mis-web/config.js @@ -11,7 +11,6 @@ */ const { envConfig, str, bool } = require("@scow/lib-config"); -const { getClusterConfigs, getSortedClusterIds } = require("@scow/config/build/cluster"); const { getMisConfig } = require("@scow/config/build/mis"); const { getCommonConfig, getSystemLanguageConfig } = require("@scow/config/build/common"); const { getClusterTextsConfig } = require("@scow/config/build/clusterTexts"); @@ -92,7 +91,6 @@ const buildRuntimeConfig = async (phase, basePath) => { const configBasePath = mockEnv ? join(__dirname, "config") : undefined; - const clusters = getClusterConfigs(configBasePath, console); const clusterTexts = getClusterTextsConfig(configBasePath, console); const uiConfig = getUiConfig(configBasePath, console); const misConfig = getMisConfig(configBasePath, console); @@ -111,7 +109,6 @@ const buildRuntimeConfig = async (phase, basePath) => { const serverRuntimeConfig = { AUTH_EXTERNAL_URL: config.AUTH_EXTERNAL_URL, AUTH_INTERNAL_URL: config.AUTH_INTERNAL_URL, - CLUSTERS_CONFIG: clusters, CLUSTER_TEXTS_CONFIG: clusterTexts, UI_CONFIG: uiConfig, DEFAULT_PRIMARY_COLOR, @@ -138,13 +135,6 @@ const buildRuntimeConfig = async (phase, basePath) => { PUBLIC_PATH: config.PUBLIC_PATH, - CLUSTERS: Object.keys(clusters).reduce((prev, curr) => { - prev[curr] = { id: curr, name: clusters[curr].displayName }; - return prev; - }, {}), - - CLUSTER_SORTED_ID_LIST: getSortedClusterIds(clusters), - ACCOUNT_NAME_PATTERN: misConfig.accountNamePattern?.regex, PORTAL_URL: config.PORTAL_DEPLOYED ? (config.PORTAL_URL || misConfig.portalUrl || "") : undefined, diff --git a/apps/mis-web/src/apis/api.mock.ts b/apps/mis-web/src/apis/api.mock.ts index fecfc214c8..9fa9d0557f 100644 --- a/apps/mis-web/src/apis/api.mock.ts +++ b/apps/mis-web/src/apis/api.mock.ts @@ -11,12 +11,14 @@ */ import { HttpError, JsonFetchResultPromiseLike } from "@ddadaal/next-typed-api-routes-runtime/lib/client"; +import { ClusterActivationStatus } from "@scow/config/build/type"; import { numberToMoney } from "@scow/lib-decimal"; import { JobInfo } from "@scow/protos/build/common/ended_job"; import type { RunningJob } from "@scow/protos/build/common/job"; import { type Account } from "@scow/protos/build/server/account"; import type { AccountUserInfo, GetUserStatusResponse } from "@scow/protos/build/server/user"; import { api } from "src/apis/api"; +import { ClusterConnectionStatus } from "src/models/cluster"; import { OperationResult } from "src/models/operationLog"; import { AccountState, ClusterAccountInfo_ImportStatus, DisplayedAccountState, PlatformRole, TenantRole, UserInfo, UserRole, UserStatus } from "src/models/User"; @@ -483,6 +485,47 @@ export const mockApi: MockApi = { }]}), getAlarmLogsCount: async () => ({ totalCount: 1 }), changeTenant: async () => null, + + getClusterConfigFiles: async () => ({ clusterConfigs: { + hpc01: { + displayName: "hpc01Name", + priority: 1, + adapterUrl: "0.0.0.0:0000", + proxyGateway: undefined, + loginNodes: [{ "address": "localhost:22222", "name": "login" }], + loginDesktop: undefined, + turboVncPath: undefined, + crossClusterFileTransfer: undefined, + hpc: { enabled: true }, + ai: { enabled: false }, + k8s: undefined, + }, + } }), + + getClustersConnectionInfo: async () => ({ results: [{ + clusterId: "hpc01", + schedulerName: "hpc", + connectionStatus: ClusterConnectionStatus.AVAILABLE, + partitions: [], + }]}), + + getClustersRuntimeInfo: async () => ({ results: [{ + clusterId: "hpc01", + activationStatus: ClusterActivationStatus.ACTIVATED, + operatorId: undefined, + operatorName: undefined, + comment: "", + }]}), + + activateCluster: async () => ({ executed: true }), + deactivateCluster: async () => ({ executed: true }), + + exportAccount: null, + exportChargeRecord: null, + exportPayRecord: null, + exportUser: null, + exportOperationLog: null, + }; export const MOCK_USER_INFO = { diff --git a/apps/mis-web/src/apis/api.ts b/apps/mis-web/src/apis/api.ts index f1cf5551ba..083c6dbff6 100644 --- a/apps/mis-web/src/apis/api.ts +++ b/apps/mis-web/src/apis/api.ts @@ -13,9 +13,12 @@ /* eslint-disable max-len */ import { apiClient } from "src/apis/client"; +import type { getClusterConfigFilesSchema } from "src/pages/api//clusterConfigsInfo"; +import type { ActivateClusterSchema } from "src/pages/api/admin/activateCluster"; import type { ChangeJobPriceSchema } from "src/pages/api/admin/changeJobPrice"; import type { ChangePasswordAsPlatformAdminSchema } from "src/pages/api/admin/changePassword"; import type { ChangeStorageQuotaSchema } from "src/pages/api/admin/changeStorage"; +import type { DeactivateClusterSchema } from "src/pages/api/admin/deactivateCluster"; import type { FetchJobsSchema } from "src/pages/api/admin/fetchJobs/fetchJobs"; import type { GetFetchJobInfoSchema } from "src/pages/api/admin/fetchJobs/getFetchInfo"; import type { SetFetchStateSchema } from "src/pages/api/admin/fetchJobs/setFetchState"; @@ -25,6 +28,8 @@ import type { GetActiveUserCountSchema } from "src/pages/api/admin/getActiveUser import type { GetAllAccountsSchema } from "src/pages/api/admin/getAllAccounts"; import type { GetAllTenantsSchema } from "src/pages/api/admin/getAllTenants"; import type { GetAllUsersSchema } from "src/pages/api/admin/getAllUsers"; +import type { GetClustersConnectionInfoSchema } from "src/pages/api/admin/getClustersConnectionInfo"; +import type { GetClustersRuntimeInfoSchema } from "src/pages/api/admin/getClustersRuntimeInfo"; import type { GetClusterUsersSchema } from "src/pages/api/admin/getClusterUsers"; import type { GetDailyChargeSchema } from "src/pages/api/admin/getDailyCharge"; import type { GetDailyPaySchema } from "src/pages/api/admin/getDailyPay"; @@ -56,6 +61,11 @@ import type { AuthCallbackSchema } from "src/pages/api/auth/callback"; import type { LogoutSchema } from "src/pages/api/auth/logout"; import type { ValidateTokenSchema } from "src/pages/api/auth/validateToken"; import type { GetUserStatusSchema } from "src/pages/api/dashboard/status"; +import type { ExportAccountSchema } from "src/pages/api/file/exportAccount"; +import type { ExportChargeRecordSchema } from "src/pages/api/file/exportChargeRecord"; +import type { ExportOperationLogSchema } from "src/pages/api/file/exportOperationLog"; +import type { ExportPayRecordSchema } from "src/pages/api/file/exportPayRecord"; +import type { ExportUserSchema } from "src/pages/api/file/exportUser"; import type { GetChargesSchema } from "src/pages/api/finance/charges"; import type { GetChargeRecordsTotalCountSchema } from "src/pages/api/finance/getChargeRecordsTotalCount"; import type { GetUsedPayTypesSchema } from "src/pages/api/finance/getUsedPayTypes"; @@ -107,23 +117,41 @@ import type { RemoveUserFromAccountSchema } from "src/pages/api/users/removeFrom import type { SetAdminSchema } from "src/pages/api/users/setAsAdmin"; import type { QueryStorageUsageSchema } from "src/pages/api/users/storageUsage"; import type { UnblockUserInAccountSchema } from "src/pages/api/users/unblockInAccount"; -import type { UnsetAdminSchema } from "src/pages/api/users/unsetAdmin"; ; +import type { UnsetAdminSchema } from "src/pages/api/users/unsetAdmin"; + export const api = { + activateCluster: apiClient.fromTypeboxRoute("PUT", "/api/admin/activateCluster"), changeJobPrice: apiClient.fromTypeboxRoute("PATCH", "/api/admin/changeJobPrice"), changePasswordAsPlatformAdmin: apiClient.fromTypeboxRoute("PATCH", "/api/admin/changePassword"), changeStorageQuota: apiClient.fromTypeboxRoute("PUT", "/api/admin/changeStorage"), + deactivateCluster: apiClient.fromTypeboxRoute("PUT", "/api/admin/deactivateCluster"), fetchJobs: apiClient.fromTypeboxRoute("POST", "/api/admin/fetchJobs/fetchJobs"), getFetchJobInfo: apiClient.fromTypeboxRoute("GET", "/api/admin/fetchJobs/getFetchInfo"), setFetchState: apiClient.fromTypeboxRoute("POST", "/api/admin/fetchJobs/setFetchState"), tenantFinancePay: apiClient.fromTypeboxRoute("POST", "/api/admin/finance/pay"), getTenantPayments: apiClient.fromTypeboxRoute("GET", "/api/admin/finance/payments"), + getActiveUserCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getActiveUserCount"), getAllAccounts: apiClient.fromTypeboxRoute("GET", "/api/admin/getAllAccounts"), getAllTenants: apiClient.fromTypeboxRoute("GET", "/api/admin/getAllTenants"), getAllUsers: apiClient.fromTypeboxRoute("GET", "/api/admin/getAllUsers"), getClusterUsers: apiClient.fromTypeboxRoute("GET", "/api/admin/getClusterUsers"), + getClustersConnectionInfo: apiClient.fromTypeboxRoute("GET", "/api/admin/getClustersConnectionInfo"), + getClustersRuntimeInfo: apiClient.fromTypeboxRoute("GET", "/api/admin/getClustersRuntimeInfo"), + getDailyCharge: apiClient.fromTypeboxRoute("GET", "/api/admin/getDailyCharge"), + getDailyPay: apiClient.fromTypeboxRoute("GET", "/api/admin/getDailyPay"), + getJobTotalCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getJobTotalCount"), + getMisUsageCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getMisUsageCount"), + getNewJobCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getNewJobCount"), + getNewUserCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getNewUserCount"), getPlatformUsersCounts: apiClient.fromTypeboxRoute("GET", "/api/admin/getPlatformUsersCounts"), + getPortalUsageCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getPortalUsageCount"), + getStatisticInfo: apiClient.fromTypeboxRoute("GET", "/api/admin/getStatisticInfo"), getTenantUsers: apiClient.fromTypeboxRoute("GET", "/api/admin/getTenantUsers"), + getTopChargeAccount: apiClient.fromTypeboxRoute("GET", "/api/admin/getTopChargeAccount"), + getTopPayAccount: apiClient.fromTypeboxRoute("GET", "/api/admin/getTopPayAccount"), + getTopSubmitJobUser: apiClient.fromTypeboxRoute("GET", "/api/admin/getTopSubmitJobUser"), + getUsersWithMostJobSubmissions: apiClient.fromTypeboxRoute("GET", "/api/admin/getUsersWithMostJobSubmissions"), importUsers: apiClient.fromTypeboxRoute("POST", "/api/admin/importUsers"), getAlarmDbId: apiClient.fromTypeboxRoute("GET", "/api/admin/monitor/getAlarmDbId"), getAlarmLogs: apiClient.fromTypeboxRoute("GET", "/api/admin/monitor/getAlarmLogs"), @@ -136,20 +164,16 @@ export const api = { syncBlockStatus: apiClient.fromTypeboxRoute("PUT", "/api/admin/synchronize/syncBlockStatus"), unsetPlatformRole: apiClient.fromTypeboxRoute("PUT", "/api/admin/unsetPlatformRole"), unsetTenantRole: apiClient.fromTypeboxRoute("PUT", "/api/admin/unsetTenantRole"), - getNewUserCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getNewUserCount"), - getActiveUserCount:apiClient.fromTypeboxRoute("GET", "/api/admin/getActiveUserCount"), - getTopChargeAccount: apiClient.fromTypeboxRoute("GET", "/api/admin/getTopChargeAccount"), - getDailyCharge: apiClient.fromTypeboxRoute("GET", "/api/admin/getDailyCharge"), - getTopPayAccount: apiClient.fromTypeboxRoute("GET", "/api/admin/getTopPayAccount"), - getDailyPay: apiClient.fromTypeboxRoute("GET", "/api/admin/getDailyPay"), - getPortalUsageCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getPortalUsageCount"), - getMisUsageCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getMisUsageCount"), - getStatisticInfo: apiClient.fromTypeboxRoute("GET", "/api/admin/getStatisticInfo"), - getJobTotalCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getJobTotalCount"), authCallback: apiClient.fromTypeboxRoute("GET", "/api/auth/callback"), logout: apiClient.fromTypeboxRoute("DELETE", "/api/auth/logout"), validateToken: apiClient.fromTypeboxRoute("GET", "/api/auth/validateToken"), + getClusterConfigFiles: apiClient.fromTypeboxRoute("GET", "/api//clusterConfigsInfo"), getUserStatus: apiClient.fromTypeboxRoute("GET", "/api/dashboard/status"), + exportAccount: apiClient.fromTypeboxRoute("GET", "/api/file/exportAccount"), + exportChargeRecord: apiClient.fromTypeboxRoute("GET", "/api/file/exportChargeRecord"), + exportOperationLog: apiClient.fromTypeboxRoute("GET", "/api/file/exportOperationLog"), + exportPayRecord: apiClient.fromTypeboxRoute("GET", "/api/file/exportPayRecord"), + exportUser: apiClient.fromTypeboxRoute("GET", "/api/file/exportUser"), getCharges: apiClient.fromTypeboxRoute("GET", "/api/finance/charges"), getChargeRecordsTotalCount: apiClient.fromTypeboxRoute("GET", "/api/finance/getChargeRecordsTotalCount"), getUsedPayTypes: apiClient.fromTypeboxRoute("GET", "/api/finance/getUsedPayTypes"), @@ -172,11 +196,8 @@ export const api = { getJobInfo: apiClient.fromTypeboxRoute("GET", "/api/job/jobInfo"), queryJobTimeLimit: apiClient.fromTypeboxRoute("GET", "/api/job/queryJobTimeLimit"), getRunningJobs: apiClient.fromTypeboxRoute("GET", "/api/job/runningJobs"), - getTopSubmitJobUser: apiClient.fromTypeboxRoute("GET", "/api/admin/getTopSubmitJobUser"), - getUsersWithMostJobSubmissions: apiClient.fromTypeboxRoute("GET", "/api/admin/getUsersWithMostJobSubmissions"), - getNewJobCount: apiClient.fromTypeboxRoute("GET", "/api/admin/getNewJobCount"), - getOperationLogs: apiClient.fromTypeboxRoute("GET", "/api/log/getOperationLog"), getCustomEventTypes: apiClient.fromTypeboxRoute("GET", "/api/log/getCustomEventTypes"), + getOperationLogs: apiClient.fromTypeboxRoute("GET", "/api/log/getOperationLog"), changeEmail: apiClient.fromTypeboxRoute("PATCH", "/api/profile/changeEmail"), changePassword: apiClient.fromTypeboxRoute("PATCH", "/api/profile/changePassword"), checkPassword: apiClient.fromTypeboxRoute("GET", "/api/profile/checkPassword"), @@ -186,15 +207,16 @@ export const api = { blockAccount: apiClient.fromTypeboxRoute("PUT", "/api/tenant/blockAccount"), changePasswordAsTenantAdmin: apiClient.fromTypeboxRoute("PATCH", "/api/tenant/changePassword"), createTenant: apiClient.fromTypeboxRoute("POST", "/api/tenant/create"), - createTenantWithExistingUserAsAdmin: apiClient.fromTypeboxRoute("POST", "/api/tenant/createTenantWithExistingUserAsAdmin"), createAccount: apiClient.fromTypeboxRoute("POST", "/api/tenant/createAccount"), + createTenantWithExistingUserAsAdmin: apiClient.fromTypeboxRoute("POST", "/api/tenant/createTenantWithExistingUserAsAdmin"), getAccounts: apiClient.fromTypeboxRoute("GET", "/api/tenant/getAccounts"), getTenants: apiClient.fromTypeboxRoute("GET", "/api/tenant/getTenants"), - setDefaultAccountBlockThreshold: apiClient.fromTypeboxRoute("PUT", "/api/tenant/setDefaultAccountBlockThreshold"), setBlockThreshold: apiClient.fromTypeboxRoute("PUT", "/api/tenant/setBlockThreshold"), + setDefaultAccountBlockThreshold: apiClient.fromTypeboxRoute("PUT", "/api/tenant/setDefaultAccountBlockThreshold"), unblockAccount: apiClient.fromTypeboxRoute("PUT", "/api/tenant/unblockAccount"), addUserToAccount: apiClient.fromTypeboxRoute("POST", "/api/users/addToAccount"), blockUserInAccount: apiClient.fromTypeboxRoute("PUT", "/api/users/blockInAccount"), + changeTenant: apiClient.fromTypeboxRoute("PUT", "/api/users/changeTenant"), createUser: apiClient.fromTypeboxRoute("POST", "/api/users/create"), getAccountUsers: apiClient.fromTypeboxRoute("GET", "/api/users"), cancelJobChargeLimit: apiClient.fromTypeboxRoute("DELETE", "/api/users/jobChargeLimit/cancel"), @@ -204,5 +226,4 @@ export const api = { queryStorageUsage: apiClient.fromTypeboxRoute("GET", "/api/users/storageUsage"), unblockUserInAccount: apiClient.fromTypeboxRoute("PUT", "/api/users/unblockInAccount"), unsetAdmin: apiClient.fromTypeboxRoute("PUT", "/api/users/unsetAdmin"), - changeTenant: apiClient.fromTypeboxRoute("PUT", "/api/users/changeTenant"), }; diff --git a/apps/mis-web/src/auth/server.ts b/apps/mis-web/src/auth/server.ts index f53a9200c0..3d1f996d7e 100644 --- a/apps/mis-web/src/auth/server.ts +++ b/apps/mis-web/src/auth/server.ts @@ -48,7 +48,9 @@ export type SSRProps = { export const ssrAuthenticate = (check: Check) => async (req: NextPageContext["req"]) => { - return await checkCookie(check, req); + // return await checkCookie(check, req); + const result = await checkCookie(check, req); + return result; }; export const authenticate = (check: Check) => diff --git a/apps/mis-web/src/components/ClusterSelector.tsx b/apps/mis-web/src/components/ClusterSelector.tsx index 9e6e6874fc..c837c2d19b 100644 --- a/apps/mis-web/src/components/ClusterSelector.tsx +++ b/apps/mis-web/src/components/ClusterSelector.tsx @@ -12,30 +12,42 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; import { Select } from "antd"; +import { useStore } from "simstate"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; -import { Cluster, publicConfig } from "src/utils/config"; - +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; interface Props { value?: Cluster[]; onChange?: (clusters: Cluster[]) => void; + // is using config clusters or not + // true: use config clusters + // false or not exist： use current activated clusters from db + isUsingAllConfigClusters?: boolean; } + const p = prefix("component.others."); -export const ClusterSelector: React.FC = ({ value, onChange }) => { +export const ClusterSelector: React.FC = ({ value, onChange, isUsingAllConfigClusters }) => { const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters, clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const clusters = isUsingAllConfigClusters ? publicConfigClusters : activatedClusters; + + const sortedIds = + clusterSortedIdList.filter((id) => Object.keys(clusters)?.includes(id)); + return ( onChange?.({ id: value, name: publicConfig.CLUSTERS[value].name })} + onChange={(value) => onChange?.({ id: value, name: activatedClusters[value].name })} options={ (label ? [{ value: label, label, disabled: true }] : []) - .concat(publicConfig.CLUSTER_SORTED_ID_LIST.map((x) => ({ + .concat(sortedIds.map((x) => ({ value: x, - label: getI18nConfigCurrentText(publicConfig.CLUSTERS[x].name, languageId), + label: getI18nConfigCurrentText(activatedClusters[x]?.name, languageId), disabled: false, }))) } diff --git a/apps/mis-web/src/components/DeactivateClusterModal.tsx b/apps/mis-web/src/components/DeactivateClusterModal.tsx new file mode 100644 index 0000000000..6aa11bfe9d --- /dev/null +++ b/apps/mis-web/src/components/DeactivateClusterModal.tsx @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { validateDataConsistency } from "@scow/lib-web/build/utils/form"; +import { Divider, Form, Input, Modal } from "antd"; +import { useState } from "react"; +import { ModalLink } from "src/components/ModalLink"; +import { prefix, useI18n, useI18nTranslate } from "src/i18n"; + +interface Props { + clusterId: string; + clusterName: string; + onClose: () => void; + onComplete: (confirmedClusterId: string, comment: string) => Promise; + open: boolean; +} + +interface FormProps { + confirmedClusterId: string; + confirmedClusterName: string; + comment: string; +} +const p = prefix("page.admin.resourceManagement.clusterManagement.deactivateModal."); + +const DeactivateClusterModal: React.FC = ({ clusterId, clusterName, onClose, onComplete, open }) => { + + const tArgs = useI18nTranslate(); + + const [form] = Form.useForm(); + const [loading, setLoading] = useState(false); + + const onOK = async () => { + const { confirmedClusterId, comment } = await form.validateFields(); + setLoading(true); + await onComplete(confirmedClusterId, comment) + .then(() => { + form.resetFields(); + onClose(); + }) + .finally(() => setLoading(false)); + }; + + const languageId = useI18n().currentLanguage.id; + + return ( + + +

+ {tArgs(p("content"), [ + {clusterId}, + {clusterName}, + ])} +

+

{tArgs(p("contentInputNotice"))}

+

{tArgs(p("contentAttention"))}

+ +
+ + e.preventDefault()} /> + + + e.preventDefault()} /> + + + + +
+ + ); +}; +export const DeactivateClusterModalLink = ModalLink(DeactivateClusterModal); diff --git a/apps/mis-web/src/components/JobBillingTable.tsx b/apps/mis-web/src/components/JobBillingTable.tsx index 6f23b6686a..b30a14d2ff 100644 --- a/apps/mis-web/src/components/JobBillingTable.tsx +++ b/apps/mis-web/src/components/JobBillingTable.tsx @@ -13,8 +13,9 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; import { Table } from "antd"; import { ColumnsType } from "antd/es/table"; +import { useStore } from "simstate"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { AmountStrategyDescriptionsItem } from "./AmonutStrategyDescriptionsItem"; @@ -60,6 +61,8 @@ export const JobBillingTable: React.FC = ({ data, loading, isUserPartitio const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { activatedClusters } = useStore(ClusterInfoStore); + const clusterTotalQosCounts = data && data.length ? data.reduce((totalQosCounts: { [cluster: string]: number }, item) => { const { cluster } = item; @@ -74,7 +77,7 @@ export const JobBillingTable: React.FC = ({ data, loading, isUserPartitio const columns: ColumnsType = [ ...(isUserPartitionsPage ? [] : [ { dataIndex: "cluster", title: t(pCommon("cluster")), key: "index", render: (_, r) => ({ - children: getI18nConfigCurrentText(publicConfig.CLUSTERS[r.cluster]?.name, languageId) ?? r.cluster, + children: getI18nConfigCurrentText(activatedClusters[r.cluster]?.name, languageId) ?? r.cluster, props: { rowSpan: r.clusterItemIndex === 0 && clusterTotalQosCounts ? clusterTotalQosCounts[r.cluster] : 0 }, }) }, ]) diff --git a/apps/portal-server/src/utils/error.ts b/apps/mis-web/src/components/errorPages/ClusterNotAvailablePage.tsx similarity index 57% rename from apps/portal-server/src/utils/error.ts rename to apps/mis-web/src/components/errorPages/ClusterNotAvailablePage.tsx index 5fe873d14a..787cbb54b6 100644 --- a/apps/portal-server/src/utils/error.ts +++ b/apps/mis-web/src/components/errorPages/ClusterNotAvailablePage.tsx @@ -10,12 +10,22 @@ * See the Mulan PSL v2 for more details. */ -import { MetadataValue } from "@grpc/grpc-js"; +import { Result } from "antd"; +import { useI18nTranslateToString } from "src/i18n"; +import { Head } from "src/utils/head"; -export const scowErrorMetadata = (code: string, extra?: Record) => { - return { - IS_SCOW_ERROR: "1", - SCOW_ERROR_CODE: code, - ...extra, - }; +export const ClusterNotAvailablePage = () => { + + const t = useI18nTranslateToString(); + + return ( + <> + + + + ); }; diff --git a/apps/mis-web/src/i18n/en.ts b/apps/mis-web/src/i18n/en.ts index 07dfcfb180..7db5f2d295 100644 --- a/apps/mis-web/src/i18n/en.ts +++ b/apps/mis-web/src/i18n/en.ts @@ -113,6 +113,8 @@ export default { exportNoDataErrorMsg: "Export is empty, please reselect", blockThresholdAmount: "Block Threshold Amount", other: "Other", + noAvailableClusters: "There are currently no available clusters." + + " Please try again later or contact the administrator.", }, dashboard: { title: "Dashboard", @@ -180,6 +182,8 @@ export default { systemDebug: "Platform Operation", statusSynchronization: "Block Status Synchronization", jobSynchronization: "Jobs Synchronization", + resourceManagement: "Resource Management", + clusterManagement: "Cluster Management", accountList: "Accounts", clusterMonitor: "Monitor", resourceStatus: "Status", @@ -766,6 +770,8 @@ export default { pageNotExist: "The page you requested does not exist.", serverWrong: "Server Error", sorry: "Sorry, there was a server error. Please refresh and try again.", + clusterNotAvailable: "The cluster you are currently accessing is unavailable or there are no available clusters. " + + " Please try again later or contact the administrator.", }, others: { seeDetails: "For details, please refer to the documentation", @@ -811,6 +817,10 @@ export default { + "synchronized with the modifications.", adapterConnErrorContent: "The {} cluster is currently unreachable. Please try again later. ", effectErrorMessage: "Server error occurred!", + noActivatedClusters: "No available clusters. Please try again after refreshing the page.", + notExistInActivatedClusters: "The cluster(s) being queried may have been deactivated. " + + "Please try again after refreshing the page.", + noClusters: "Unable to find cluster configuration files. Please contact the system administrator.", }, profile: { index: { @@ -1024,6 +1034,49 @@ export default { syncJobNow: "Sync Now", }, }, + resourceManagement: { + clusterManagement: { + title: "Cluster Management", + clusterFilter: "Cluster", + table: { + clusterName: "Cluster Name", + nodesCount: "Total Nodes", + cpusCount: "Total CPU Cores", + gpusCount: "Total GPU Cards", + totalMemMb: "Total Memory Capacity", + clusterState: "Cluster State", + errorState: "Error", + deactivatedState: "Deactivated", + normalState: "Normal", + operator: "Operator", + lastOperatedTime: "Last Operation Time", + comment: "Comment", + operation: "Operation", + activate: "Activate", + deactivate: "Deactivate", + }, + activateModal: { + title: "Activate Cluster", + content: "Please confirm if you want to activate the cluster with Cluster ID {}, named {}?", + contentAttention: "Attention: Please manually synchronize platform data after activation!", + successMessage: "The cluster has been activated.", + failureMessage: "Failed to activate the cluster. The cluster may have been activated.", + }, + deactivateModal: { + title: "Deactivate Cluster", + content: "Please confirm if you want to deactivate the cluster with Cluster ID {}, named {}?", + contentInputNotice: "If you confirm the deactivation of the cluster, " + + "please re-enter the cluster ID and name below.", + contentAttention: "Attention: After deactivation, the cluster will not be available, " + + "and all data updates for the cluster will cease!", + clusterNameForm: "Cluster Name", + clusterIdForm: "Cluster ID", + comment: "Deactivation Comment", + successMessage: "The cluster has been deactivated.", + failureMessage: "Failed to deactivate the cluster. The cluster may have been deactivated.", + }, + }, + }, finance: { pay: { tenantCharge: "Tenant Charge", @@ -1156,6 +1209,8 @@ export default { setAccountDefaultBlockThreshold: "Set Default Account Block Threshold", userChangeTenant: "User Change Tenant", customEvent: "Custom Operation Event", + activateCluster: "Activate Cluster", + deactivateCluster: "Deactivate Cluster", }, operationDetails: { login: "User Login", @@ -1234,6 +1289,8 @@ export default { setAccountDefaultBlockThreshold: "Set the default block threshold of accounts in Tenant {} to {}", unsetAccountBlockThreshold: "Reset the block threshold of account {} to default", userChangeTenant: "User {} changes from tenant {} to tenant {}", + activateCluster: "User {} activates the Cluster: {}", + deactivateCluster: "User {} deactivates the Cluster: {}", }, }, userRoles: { diff --git a/apps/mis-web/src/i18n/zh_cn.ts b/apps/mis-web/src/i18n/zh_cn.ts index 24ad14ec23..ad907817cc 100644 --- a/apps/mis-web/src/i18n/zh_cn.ts +++ b/apps/mis-web/src/i18n/zh_cn.ts @@ -112,6 +112,8 @@ export default { exportNoDataErrorMsg: "导出为空，请重新选择", blockThresholdAmount: "封锁阈值", other: "其他", + noAvailableClusters: "当前没有可用集群。" + + "请稍后再试或联系管理员。", }, dashboard: { title: "仪表盘", @@ -179,6 +181,8 @@ export default { systemDebug: "平台调试", statusSynchronization: "封锁状态同步", jobSynchronization: "作业信息同步", + resourceManagement: "资源管理", + clusterManagement: "集群管理", accountList: "账户列表", clusterMonitor: "集群监控", resourceStatus: "资源状态", @@ -764,6 +768,8 @@ export default { pageNotExist:"您所请求的页面不存在。", serverWrong:"服务器出错", sorry:"对不起，服务器出错。请刷新重试。", + clusterNotAvailable: "当前正在访问的集群不可用或没有可用集群。" + + "请稍后再试或联系管理员。", }, others:{ seeDetails:"细节请查阅文档", @@ -809,6 +815,10 @@ export default { adapterConnErrorContent: "{} 集群无法连接，请稍后重试 ", effectErrorMessage: "服务器出错啦！", + noActivatedClusters: "现在没有可用的集群，请在页面刷新后重试。", + notExistInActivatedClusters: "正在查询的集群可能已被停用，请在页面刷新后重试。", + + noClusters: "无法找到集群的配置文件，请联系管理员。", }, profile: { index: { @@ -989,7 +999,6 @@ export default { systemDebug: { slurmBlockStatus: { syncUserAccountBlockingStatus: "用户账户封锁状态同步", - alertInfo: "SCOW会定期向调度器同步SCOW数据库中账户和用户的封锁状态，您可以点击立刻同步执行一次手动同步", periodicSyncUserAccountBlockStatusInfo:"周期性同步调度器账户和用户的封锁状态", @@ -1023,6 +1032,49 @@ export default { syncJobNow: "立刻同步作业", }, }, + resourceManagement: { + clusterManagement: { + title: "集群管理", + clusterFilter: "集群", + table: { + clusterName: "集群名称", + nodesCount: "节点总数", + cpusCount: "CPU总核数", + gpusCount: "GPU总卡数", + totalMemMb: "内存总容量", + clusterState: "集群状态", + errorState: "异常", + deactivatedState: "停用", + normalState: "正常", + operator: "操作员", + lastOperatedTime: "上次启用/停用时间", + comment: "备注", + operation: "操作", + activate: "启用", + deactivate: "停用", + }, + activateModal: { + title: "启用集群", + content: "请确认是否启用集群ID是 {}，集群名称是 {} 的集群？", + contentAttention: "注意：启用后请手动同步平台数据！", + successMessage: "集群已启用", + failureMessage: "集群启用失败，集群可能已被启用", + }, + deactivateModal: { + title: "停用集群", + content: "请确认是否停用集群ID是 {}，集群名称是 {} 的集群？", + contentInputNotice: "如果确认停用集群，请在下面重复输入上述集群ID和集群名称", + + contentAttention: "注意：停用后集群将不可用，集群所有数据不再更新！", + + clusterNameForm: "集群名称", + clusterIdForm: "集群ID", + comment: "停用备注", + successMessage: "集群已停用", + failureMessage: "集群停用失败，集群可能已被停用", + }, + }, + }, finance: { pay: { tenantCharge: "租户充值", @@ -1155,6 +1207,8 @@ export default { setAccountDefaultBlockThreshold: "设置账户默认封锁阈值", userChangeTenant: "用户切换租户", customEvent: "自定义操作行为", + activateCluster: "启用集群", + deactivateCluster: "停用集群", }, operationDetails: { login: "用户登录", @@ -1233,6 +1287,8 @@ export default { setAccountDefaultBlockThreshold: "设置租户{}的默认账户封锁阈值为{}", unsetAccountBlockThreshold: "账户{}恢复使用默认封锁阈值", userChangeTenant: "用户{}切换租户，从租户{}切换到租户{}", + activateCluster: "用户{}启用集群：{}", + deactivateCluster: "用户{}停用集群：{}", }, }, userRoles: { diff --git a/apps/mis-web/src/layouts/routes.tsx b/apps/mis-web/src/layouts/routes.tsx index ad8819632c..fdc60a1a54 100644 --- a/apps/mis-web/src/layouts/routes.tsx +++ b/apps/mis-web/src/layouts/routes.tsx @@ -12,6 +12,8 @@ import { AccountBookOutlined, AlertOutlined, BookOutlined, CloudServerOutlined, + ClusterOutlined, + ControlOutlined, DashboardOutlined, InfoOutlined, LineChartOutlined, LinkOutlined, LockOutlined, MoneyCollectOutlined, MonitorOutlined, PartitionOutlined, PlusOutlined, PlusSquareOutlined, ProfileOutlined, @@ -131,6 +133,19 @@ export const platformAdminRoutes: (platformRoles: PlatformRole[], t: TransType) }, ], }, + ...(platformRoles.includes(PlatformRole.PLATFORM_ADMIN) ? [{ + Icon: ControlOutlined, + text: t(pPlatform("resourceManagement")), + path: "/admin/resource", + clickable: false, + children: [ + { + Icon: ClusterOutlined, + text: t("layouts.route.platformManagement.clusterManagement"), + path: "/admin/resource/clusterManagement", + }, + ], + }] : []), ...(platformRoles.includes(PlatformRole.PLATFORM_ADMIN) && (publicConfig.CLUSTER_MONITOR.resourceStatus.enabled || publicConfig.CLUSTER_MONITOR.alarmLogs.enabled) ? [{ Icon: MonitorOutlined, diff --git a/apps/mis-web/src/models/cluster.ts b/apps/mis-web/src/models/cluster.ts new file mode 100644 index 0000000000..97a1d27b48 --- /dev/null +++ b/apps/mis-web/src/models/cluster.ts @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { Static, Type } from "@sinclair/typebox"; +import { ValueOf } from "next/dist/shared/lib/constants"; + +export const Partition = Type.Object({ + name: Type.String(), + memMb: Type.Number(), + cores: Type.Number(), + gpus: Type.Number(), + nodes: Type.Number(), + qos: Type.Optional(Type.Array(Type.String())), + comment: Type.Optional(Type.String()), +}); +export type Partition = Static; + +export const ClusterConnectionStatus = { + AVAILABLE: 0, + ERROR: 1, +} as const; + +export type ClusterConnectionStatus = ValueOf; + +export const ClusterConnectionInfoSchema = Type.Object({ + clusterId: Type.String(), + connectionStatus: Type.Enum(ClusterConnectionStatus), + schedulerName: Type.Optional(Type.String()), + partitions: Type.Array(Partition), +}); + +export type ClusterConnectionInfo = Static; diff --git a/apps/mis-web/src/models/job.ts b/apps/mis-web/src/models/job.ts index 8aaf756fc8..db7d97e46b 100644 --- a/apps/mis-web/src/models/job.ts +++ b/apps/mis-web/src/models/job.ts @@ -15,7 +15,7 @@ import { Static, Type } from "@sinclair/typebox"; import dayjs from "dayjs"; import { Lang } from "react-typed-i18n"; import en from "src/i18n/en"; -import type { Cluster } from "src/utils/config"; +import type { Cluster } from "src/utils/cluster"; export type RunningJobInfo = RunningJob & { cluster: Cluster; runningOrQueueTime: string }; diff --git a/apps/mis-web/src/models/operationLog.ts b/apps/mis-web/src/models/operationLog.ts index f41b5cb53c..ba3947c28d 100644 --- a/apps/mis-web/src/models/operationLog.ts +++ b/apps/mis-web/src/models/operationLog.ts @@ -86,6 +86,8 @@ export const OperationType: OperationTypeEnum = { setAccountBlockThreshold: "setAccountBlockThreshold", setAccountDefaultBlockThreshold: "setAccountDefaultBlockThreshold", userChangeTenant: "userChangeTenant", + activateCluster: "activateCluster", + deactivateCluster: "deactivateCluster", customEvent: "customEvent", }; @@ -202,6 +204,8 @@ export const getOperationTypeTexts = (t: OperationTextsTransType): { [key in Lib setAccountBlockThreshold: t(pTypes("setAccountBlockThreshold")), setAccountDefaultBlockThreshold: t(pTypes("setAccountDefaultBlockThreshold")), userChangeTenant: t(pTypes("userChangeTenant")), + activateCluster: t(pTypes("activateCluster")), + deactivateCluster: t(pTypes("deactivateCluster")), customEvent: t(pTypes("customEvent")), }; @@ -266,6 +270,8 @@ export const OperationCodeMap: { [key in LibOperationType]: string } = { exportPayRecord: "040306", exportOperationLog: "040307", userChangeTenant: "040308", + activateCluster: "040309", + deactivateCluster: "040310", customEvent: "050001", }; @@ -462,6 +468,14 @@ export const getOperationDetail = ( [operationEvent[logEvent].userId, operationEvent[logEvent].previousTenantName, operationEvent[logEvent].newTenantName]); + case "activateCluster": + return t(pDetails("activateCluster"), + [operationEvent[logEvent].userId, + operationEvent[logEvent].clusterId]); + case "deactivateCluster": + return t(pDetails("deactivateCluster"), + [operationEvent[logEvent].userId, + operationEvent[logEvent].clusterId]); case "customEvent": const c = operationEvent[logEvent]?.content; return getI18nCurrentText(c, languageId); diff --git a/apps/mis-web/src/pageComponents/admin/ClusterManagementTable.tsx b/apps/mis-web/src/pageComponents/admin/ClusterManagementTable.tsx new file mode 100644 index 0000000000..39975096db --- /dev/null +++ b/apps/mis-web/src/pageComponents/admin/ClusterManagementTable.tsx @@ -0,0 +1,268 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { ExclamationCircleOutlined } from "@ant-design/icons"; +import { ClusterActivationStatus } from "@scow/config/build/type"; +import { formatDateTime } from "@scow/lib-web/build/utils/datetime"; +import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; +import { App, Button, Form, Space, Table, Tag } from "antd"; +import React, { useMemo, useState } from "react"; +import { useStore } from "simstate"; +import { api } from "src/apis"; +import { ClusterSelector } from "src/components/ClusterSelector"; +import { DeactivateClusterModalLink } from "src/components/DeactivateClusterModal"; +import { FilterFormContainer } from "src/components/FilterFormContainer"; +import { prefix, useI18n, useI18nTranslate } from "src/i18n"; +import { ClusterConnectionStatus } from "src/models/cluster"; +import { CombinedClusterInfo } from "src/pages/admin/resource/clusterManagement"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster, getSortedClusterValues } from "src/utils/cluster"; + +interface Props { + data?: CombinedClusterInfo[]; + isLoading: boolean; + reload: () => void; +} + + +interface FilterForm { + clusters: Cluster[]; +} + +const p = prefix("page.admin.resourceManagement.clusterManagement."); +const pCommon = prefix("common."); + +export const ClusterManagementTable: React.FC = ({ + data, isLoading, reload, +}) => { + + const { message, modal } = App.useApp(); + const [form] = Form.useForm(); + + const tArgs = useI18nTranslate(); + const languageId = useI18n().currentLanguage.id; + + const { publicConfigClusters, clusterSortedIdList } = useStore(ClusterInfoStore); + + const [query, setQuery] = useState(() => { + + return { + clusters: getSortedClusterValues(publicConfigClusters, clusterSortedIdList), + }; + }); + + const filteredData = useMemo(() => { + + if (!data) return undefined; + + if (!query.clusters || query.clusters.length === 0) { + return data; + } + + const filteredValues = data + .filter((cluster) => query.clusters.some((c) => c.id === cluster.clusterId)); + + return filteredValues; + + }, [data, query]); + + + return ( +
+ + + layout="inline" + form={form} + initialValues={query} + onFinish={async () => { + setQuery(await form.validateFields()); + }} + > + + + + + + + + + + + + + + dataIndex="clusterId" + title={tArgs(p("table.clusterName"))} + render={(_, r) => { + const clusterName = publicConfigClusters[r.clusterId].name; + return getI18nConfigCurrentText(clusterName ?? r.clusterId, languageId); + }} + /> + + dataIndex="partitions" + title={tArgs(p("table.nodesCount"))} + render={(_, r) => r.partitions?.reduce((sum, p) => sum + p.nodes, 0) || 0} + /> + + dataIndex="partitions" + title={tArgs(p("table.cpusCount"))} + render={(_, r) => r.partitions?.reduce((sum, p) => sum + p.cores, 0) || 0} + /> + + dataIndex="partitions" + title={tArgs(p("table.gpusCount"))} + render={(_, r) => r.partitions?.reduce((sum, p) => sum + p.gpus, 0) || 0} + /> + + dataIndex="partitions" + title={tArgs(p("table.totalMemMb"))} + width="10%" + render={(_, r) => { + const totalMemMb = r.partitions?.reduce((sum, p) => sum + p.memMb, 0) || 0; + return `${totalMemMb} MB`; + }} + /> + + dataIndex="connectionStatus" + title={tArgs(p("table.clusterState"))} + render={(_, r) => ( + r.connectionStatus === ClusterConnectionStatus.ERROR ? ( + {tArgs(p("table.errorState"))} + ) : ( + r.activationStatus === ClusterActivationStatus.DEACTIVATED ? + {tArgs(p("table.deactivatedState"))} : + {tArgs(p("table.normalState"))} + ) + )} + /> + + dataIndex="operatorId" + title={tArgs(p("table.operator"))} + width="20%" + render={(_, r) => { + return r.operatorId ? `${r.operatorName}（ID: ${r.operatorId}）` : ""; + }} + /> + + dataIndex="updateTime" + title={tArgs(p("table.lastOperatedTime"))} + width="15%" + render={(_, r) => formatDateTime(r.updateTime)} + /> + + dataIndex="deactivationComment" + ellipsis + title={tArgs(p("table.comment"))} + /> + + title={tArgs(p("table.operation"))} + fixed="right" + width="10%" + render={(_, r) => { + const clusterName + = getI18nConfigCurrentText(publicConfigClusters[r.clusterId].name, languageId); + return ( + <> + {/* TODO: 暂时只对门户系统（HPC）中的集群增加启用和停用功能 */} + { + !r.hpcEnabled && ( + <> + -- + + ) + } + { + r.hpcEnabled && r.activationStatus === ClusterActivationStatus.DEACTIVATED + && r.connectionStatus === ClusterConnectionStatus.AVAILABLE + && ( + <> + { + + modal.confirm({ + title: tArgs(p("activateModal.title")), + icon: , + content: ( + <> +

+ {tArgs(p("activateModal.content"), [ + {r.clusterId}, + {clusterName}, + ])}, +

+

{tArgs(p("activateModal.contentAttention"))}

+ + ), + onOk: async () => { + await api.activateCluster({ + body: { + clusterId: r.clusterId, + }, + }) + .then((res) => { + if (res.executed) { + message.success(tArgs(p("activateModal.successMessage"))); + reload(); + } else { + message.error(res.reason || tArgs(p("activateModal.failureMessage"))); + reload(); + } + }); + }, + }); + + }} + > + {tArgs(p("table.activate"))} + + + ) + } + { r.hpcEnabled && r.activationStatus === ClusterActivationStatus.ACTIVATED && ( + <> + { + + return await api.deactivateCluster({ body:{ + clusterId: confirmedClusterId, + deactivationComment, + } }).then((res) => { + if (res.executed) { + message.success(tArgs(p("deactivateModal.successMessage"))); + reload(); + } else { + message.error(tArgs(p("deactivateModal.failureMessage"))); + reload(); + } + }); + + }} + > + {tArgs(p("table.deactivate"))} + + + ) + } + + ); }} + /> +
+
+ ); +}; diff --git a/apps/mis-web/src/pageComponents/admin/ImportUsersTable.tsx b/apps/mis-web/src/pageComponents/admin/ImportUsersTable.tsx index 6fe8303fb6..a74ce204d5 100644 --- a/apps/mis-web/src/pageComponents/admin/ImportUsersTable.tsx +++ b/apps/mis-web/src/pageComponents/admin/ImportUsersTable.tsx @@ -20,11 +20,11 @@ import { useAsync } from "react-async"; import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { FilterFormContainer } from "src/components/FilterFormContainer"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { ClusterAccountInfo_ImportStatus } from "src/models/User"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; const p = prefix("pageComp.admin.ImportUsersTable."); const pCommon = prefix("common."); @@ -37,12 +37,20 @@ export const ImportUsersTable: React.FC = () => { const qs = useQuerystring(); - const defaultClusterStore = useStore(DefaultClusterStore); + const { activatedClusters, defaultCluster } = useStore(ClusterInfoStore); + + if (!defaultCluster && Object.keys(activatedClusters).length === 0) { + return ; + } const clusterParam = queryToString(qs.cluster); - const cluster = (publicConfig.CLUSTERS[clusterParam] - ? publicConfig.CLUSTERS[clusterParam] - : defaultClusterStore.cluster); + const cluster = (activatedClusters[clusterParam] + ? activatedClusters[clusterParam] + : defaultCluster); + + if (!cluster) { + return ; + } const [form] = Form.useForm<{ whitelist: boolean}>(); diff --git a/apps/mis-web/src/pageComponents/dashboard/JobsSection.tsx b/apps/mis-web/src/pageComponents/dashboard/JobsSection.tsx index 1be32f8de3..d2dd119cda 100644 --- a/apps/mis-web/src/pageComponents/dashboard/JobsSection.tsx +++ b/apps/mis-web/src/pageComponents/dashboard/JobsSection.tsx @@ -13,13 +13,14 @@ import Link from "next/link"; import React, { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { Section } from "src/components/Section"; import { Localized, useI18nTranslateToString } from "src/i18n"; import { RunningJobInfo } from "src/models/job"; import { RunningJobInfoTable } from "src/pageComponents/job/RunningJobTable"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { User } from "src/stores/UserStore"; -import { publicConfig } from "src/utils/config"; interface Props { @@ -28,20 +29,24 @@ interface Props { export const JobsSection: React.FC = ({ user }) => { + const { clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const promiseFn = useCallback(() => { - return Promise.all(publicConfig.CLUSTER_SORTED_ID_LIST.map(async (clusterId) => { + return Promise.all(clusterSortedIdList + .filter((clusterId) => Object.keys(activatedClusters).find((x) => x === clusterId)) + .map(async (clusterId) => { - const { id, name } = publicConfig.CLUSTERS[clusterId]; + const { id, name } = activatedClusters[clusterId]; - return api.getRunningJobs({ - query: { - cluster: id, - userId: user.identityId, - }, - }) - .then(({ results }) => results.map((x) => RunningJobInfo.fromGrpc(x, { id, name }))) - .catch(() => [] as RunningJobInfo[]); - }, [])).then((x) => x.flat()); + return api.getRunningJobs({ + query: { + cluster: id, + userId: user.identityId, + }, + }) + .then(({ results }) => results.map((x) => RunningJobInfo.fromGrpc(x, { id, name }))) + .catch(() => [] as RunningJobInfo[]); + }, [])).then((x) => x.flat()); }, [user.identityId]); const { data, isLoading, reload } = useAsync({ promiseFn }); diff --git a/apps/mis-web/src/pageComponents/dashboard/StorageCard.tsx b/apps/mis-web/src/pageComponents/dashboard/StorageCard.tsx index a2cb93c826..cee0f9d3f0 100644 --- a/apps/mis-web/src/pageComponents/dashboard/StorageCard.tsx +++ b/apps/mis-web/src/pageComponents/dashboard/StorageCard.tsx @@ -15,11 +15,12 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLangua import { Progress, Space } from "antd"; import React, { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { DisabledA } from "src/components/DisabledA"; import { StatCard } from "src/components/StatCard"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { styled } from "styled-components"; interface Props { @@ -59,13 +60,15 @@ export const StorageCard: React.FC = ({ const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); + const { data, isLoading, run } = useAsync({ deferFn: useCallback(async () => api.queryStorageUsage({ query: { cluster } }), [cluster]), }); return ( diff --git a/apps/mis-web/src/pageComponents/init/InitJobBillingTable.tsx b/apps/mis-web/src/pageComponents/init/InitJobBillingTable.tsx index 232d387c32..11e24ba99d 100644 --- a/apps/mis-web/src/pageComponents/init/InitJobBillingTable.tsx +++ b/apps/mis-web/src/pageComponents/init/InitJobBillingTable.tsx @@ -13,9 +13,11 @@ import { Typography } from "antd"; import { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { ManageJobBillingTable } from "src/pageComponents/job/ManageJobBillingTable"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; const p = prefix("pageComp.init.initJobBillingTable."); const pCommon = prefix("common."); @@ -23,10 +25,12 @@ const pCommon = prefix("common."); export const InitJobBillingTable: React.FC = () => { const t = useI18nTranslateToString(); + const { clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const currentActivatedClusterIds = Object.keys(activatedClusters); const { data, isLoading, reload } = useAsync({ promiseFn: useCallback(async () => { return await api.getBillingItems({ - query: { tenant: undefined, activeOnly: false }, + query: { tenant: undefined, activeOnly: false, currentActivatedClusterIds, clusterSortedIdList }, }); }, []) }); @@ -36,6 +40,9 @@ export const InitJobBillingTable: React.FC = () => { {t(p("set"))} {t(pCommon("fresh"))} + { currentActivatedClusterIds.length === 0 && +
{t("common.noAvailableClusters")}
+ } = ({ data, loading, tenant const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); + const clusterTotalQosCounts = data && data.length ? data.reduce((totalQosCounts: { [cluster: string]: number }, item) => { const { cluster } = item; @@ -116,7 +119,7 @@ export const EditableJobBillingTable: React.FC = ({ data, loading, tenant const columns: ColumnsType = [ { dataIndex: "cluster", title: t(pCommon("cluster")), key: "index", render: (_, r) => ({ - children: getI18nConfigCurrentText(publicConfig.CLUSTERS[r.cluster]?.name, languageId) ?? r.cluster, + children: getI18nConfigCurrentText(publicConfigClusters[r.cluster]?.name, languageId) ?? r.cluster, props: { rowSpan: r.clusterItemIndex === 0 && clusterTotalQosCounts ? clusterTotalQosCounts[r.cluster] : 0 }, }) }, { dataIndex: "partition", title: t(p("name")), key: "index", render: (_, r) => ({ diff --git a/apps/mis-web/src/pageComponents/job/HistoryJobDrawer.tsx b/apps/mis-web/src/pageComponents/job/HistoryJobDrawer.tsx index 75fe7c84df..7f9fe4678b 100644 --- a/apps/mis-web/src/pageComponents/job/HistoryJobDrawer.tsx +++ b/apps/mis-web/src/pageComponents/job/HistoryJobDrawer.tsx @@ -13,8 +13,10 @@ import { formatDateTime } from "@scow/lib-web/build/utils/datetime"; import { JobInfo } from "@scow/protos/build/common/ended_job"; import { Descriptions, Drawer } from "antd"; +import { useStore } from "simstate"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; -import { getClusterName } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { getClusterName } from "src/utils/cluster"; import { moneyToString } from "src/utils/money"; @@ -34,6 +36,8 @@ export const HistoryJobDrawer: React.FC = (props) => { const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); + const drawerItems = [ [t(pCommon("workId")), "biJobIndex"], [t(pCommon("clusterWorkId")), "idJob"], @@ -89,7 +93,8 @@ export const HistoryJobDrawer: React.FC = (props) => { {/* 如果是集群项展示，则根据当前语言id获取集群名称 */} {format ? - (key === "cluster" ? getClusterName(item[key], languageId) : format(item[key])) + (key === "cluster" ? + getClusterName(item[key], languageId, publicConfigClusters) : format(item[key])) : item[key] as string} ) : undefined diff --git a/apps/mis-web/src/pageComponents/job/HistoryJobTable.tsx b/apps/mis-web/src/pageComponents/job/HistoryJobTable.tsx index 1015118e6d..f8c74089f1 100644 --- a/apps/mis-web/src/pageComponents/job/HistoryJobTable.tsx +++ b/apps/mis-web/src/pageComponents/job/HistoryJobTable.tsx @@ -21,6 +21,7 @@ import { App, AutoComplete, Button, DatePicker, Divider, Form, Input, InputNumbe import dayjs from "dayjs"; import React, { useCallback, useRef, useState } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { ClusterSelector } from "src/components/ClusterSelector"; import { FilterFormContainer, FilterFormTabs } from "src/components/FilterFormContainer"; @@ -29,9 +30,9 @@ import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { JobSortBy, JobSortOrder } from "src/models/job"; import { HistoryJobDrawer } from "src/pageComponents/job/HistoryJobDrawer"; import type { GetJobInfoSchema } from "src/pages/api/job/jobInfo"; -import { getSortedClusterValues } from "src/utils/cluster"; -import type { Cluster } from "src/utils/config"; -import { getClusterName } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import type { Cluster } from "src/utils/cluster"; +import { getClusterName, getSortedClusterValues } from "src/utils/cluster"; import { moneyToString, nullableMoneyToString } from "src/utils/money"; interface FilterForm { @@ -76,12 +77,16 @@ export const JobTable: React.FC = ({ const [pageInfo, setPageInfo] = useState({ page: 1, pageSize: DEFAULT_PAGE_SIZE }); const [selectedAccountName, setSelectedAccountName] = useState(undefined); + const { publicConfigClusters, clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const sortedClusters = getSortedClusterValues(publicConfigClusters, clusterSortedIdList) + .filter((x) => Object.keys(activatedClusters).includes(x.id)); + const [query, setQuery] = useState(() => { const now = dayjs(); return { jobEndTime: [now.subtract(1, "week").startOf("day"), now.endOf("day")], jobId: undefined, - clusters: getSortedClusterValues(), + clusters: sortedClusters, accountName: typeof accountNames === "string" ? accountNames : undefined, }; }); @@ -252,6 +257,7 @@ export const JobInfoTable: React.FC = ({ const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); const [previewItem, setPreviewItem] = useState(undefined); @@ -357,7 +363,7 @@ export const JobInfoTable: React.FC = ({ title={t(pCommon("clusterName"))} width="12%" ellipsis - render={(cluster) => getClusterName(cluster, languageId)} + render={(cluster) => getClusterName(cluster, languageId, publicConfigClusters)} sorter={true} /> diff --git a/apps/mis-web/src/pageComponents/job/ManageJobBillingTable.tsx b/apps/mis-web/src/pageComponents/job/ManageJobBillingTable.tsx index fd54868237..04c4cd41dc 100644 --- a/apps/mis-web/src/pageComponents/job/ManageJobBillingTable.tsx +++ b/apps/mis-web/src/pageComponents/job/ManageJobBillingTable.tsx @@ -16,6 +16,7 @@ import { DEFAULT_PAGE_SIZE } from "@scow/lib-web/build/utils/pagination"; import { Money } from "@scow/protos/build/common/money"; import { App, Form, Input, InputNumber, Modal, Popover, Select, Space, Table, Tooltip } from "antd"; import React, { useState } from "react"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { AmountStrategyDescriptionsItem } from "src/components/AmonutStrategyDescriptionsItem"; import { CommonModalProps, ModalLink } from "src/components/ModalLink"; @@ -23,7 +24,9 @@ import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { AmountStrategy, getAmountStrategyAlgorithmDescriptions, getAmountStrategyDescription, getAmountStrategyDescriptions, getAmountStrategyText } from "src/models/job"; -import { getClusterName, publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { getClusterName } from "src/utils/cluster"; +import { publicConfig } from "src/utils/config"; import { moneyToString } from "src/utils/money"; interface Props { @@ -68,6 +71,8 @@ export const ManageJobBillingTable: React.FC = ({ data, loading, tenant, const AmountStrategyText = getAmountStrategyText(t); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); + return ( = ({ data, loading, tenant, getClusterName(cluster, languageId)} + render={(cluster) => getClusterName(cluster, languageId, publicConfigClusters)} /> @@ -209,6 +214,8 @@ const EditPriceModal: React.FC(); const [loading, setLoading] = useState(false); @@ -239,7 +246,7 @@ const EditPriceModal: React.FC{tenant ? (t(pCommon("tenant")) + tenant) : t(pCommon("platform"))} - {t(pCommon("cluster"))} {getClusterName(cluster, languageId)}， + {t(pCommon("cluster"))} {getClusterName(cluster, languageId, publicConfigClusters)}， {t(pCommon("partition"))} {partition}，QOS {qos} diff --git a/apps/mis-web/src/pageComponents/job/RunningJobDrawer.tsx b/apps/mis-web/src/pageComponents/job/RunningJobDrawer.tsx index 2ee9fcd132..53e69a12bf 100644 --- a/apps/mis-web/src/pageComponents/job/RunningJobDrawer.tsx +++ b/apps/mis-web/src/pageComponents/job/RunningJobDrawer.tsx @@ -13,9 +13,11 @@ import { formatDateTime } from "@scow/lib-web/build/utils/datetime"; import { JobInfo } from "@scow/protos/build/common/ended_job"; import { Descriptions, Drawer } from "antd"; +import { useStore } from "simstate"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { RunningJobInfo } from "src/models/job"; -import { getClusterName } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { getClusterName } from "src/utils/cluster"; interface Props { open: boolean; @@ -33,6 +35,8 @@ export const RunningJobDrawer: React.FC = ({ const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); + const drawerItems = [ [t(pCommon("cluster")), "cluster", getClusterName], [t(pCommon("workId")), "jobId"], @@ -69,7 +73,8 @@ export const RunningJobDrawer: React.FC = ({ {/* 如果是集群项展示，则根据当前语言id获取集群名称 */} {format ? - (key === "cluster" ? getClusterName(item[key].id, languageId) : format(item[key], item)) + (key === "cluster" ? + getClusterName(item[key].id, languageId, publicConfigClusters) : format(item[key], item)) : item[key]} )))} diff --git a/apps/mis-web/src/pageComponents/job/RunningJobTable.tsx b/apps/mis-web/src/pageComponents/job/RunningJobTable.tsx index d57d54b690..00e78e1a35 100644 --- a/apps/mis-web/src/pageComponents/job/RunningJobTable.tsx +++ b/apps/mis-web/src/pageComponents/job/RunningJobTable.tsx @@ -20,6 +20,7 @@ import { useAsync } from "react-async"; import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { FilterFormContainer, FilterFormTabs } from "src/components/FilterFormContainer"; import { ModalLink } from "src/components/ModalLink"; import { TableTitle } from "src/components/TableTitle"; @@ -28,8 +29,9 @@ import { runningJobId, RunningJobInfo } from "src/models/job"; import { BatchChangeJobTimeLimitButton } from "src/pageComponents/job/BatchChangeJobTimeLimitButton"; import { ChangeJobTimeLimitModal } from "src/pageComponents/job/ChangeJobTimeLimitModal"; import { RunningJobDrawer } from "src/pageComponents/job/RunningJobDrawer"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { Cluster, publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import type { Cluster } from "src/utils/cluster"; +import { publicConfig } from "src/utils/config"; interface FilterForm { @@ -56,18 +58,21 @@ export const RunningJobQueryTable: React.FC = ({ const t = useI18nTranslateToString(); - const searchType = useRef<"precision" | "range">("range"); const [selected, setSelected] = useState([]); - const defaultClusterStore = useStore(DefaultClusterStore); + const { activatedClusters, defaultCluster } = useStore(ClusterInfoStore); + + if (!defaultCluster && Object.keys(activatedClusters).length === 0) { + return ; + } const [query, setQuery] = useState(() => { return { accountName: typeof accountNames === "string" ? accountNames : undefined, jobId: undefined, - cluster: defaultClusterStore.cluster, + cluster: defaultCluster ?? Object.values(activatedClusters)[0], }; }); @@ -107,7 +112,7 @@ export const RunningJobQueryTable: React.FC = ({ // add local range filters here } - return filtered.map((x) => RunningJobInfo.fromGrpc(x, publicConfig.CLUSTERS[query.cluster.id])); + return filtered.map((x) => RunningJobInfo.fromGrpc(x, activatedClusters[query.cluster.id])); }, [data, query.jobId]); return ( diff --git a/apps/mis-web/src/pageComponents/tenant/AdminJobTable.tsx b/apps/mis-web/src/pageComponents/tenant/AdminJobTable.tsx index 58b157828d..defc149d9d 100644 --- a/apps/mis-web/src/pageComponents/tenant/AdminJobTable.tsx +++ b/apps/mis-web/src/pageComponents/tenant/AdminJobTable.tsx @@ -19,6 +19,7 @@ import { Button, DatePicker, Divider, Form, Input, InputNumber, Space, Table } f import dayjs from "dayjs"; import React, { useCallback, useMemo, useRef, useState } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { ClusterSelector } from "src/components/ClusterSelector"; import { FilterFormContainer, FilterFormTabs } from "src/components/FilterFormContainer"; @@ -27,9 +28,9 @@ import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { HistoryJobDrawer } from "src/pageComponents/job/HistoryJobDrawer"; import { JobPriceChangeModal } from "src/pageComponents/tenant/JobPriceChangeModal"; import type { GetJobFilter, GetJobInfoSchema } from "src/pages/api/job/jobInfo"; -import { getSortedClusterValues } from "src/utils/cluster"; -import type { Cluster } from "src/utils/config"; -import { getClusterName } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import type { Cluster } from "src/utils/cluster"; +import { getClusterName, getSortedClusterValues } from "src/utils/cluster"; import { moneyToString, nullableMoneyToString } from "src/utils/money"; interface PageInfo { @@ -70,6 +71,11 @@ export const AdminJobTable: React.FC = () => { const rangeSearch = useRef(true); + const { publicConfigClusters, clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const sortedClusters = getSortedClusterValues(publicConfigClusters, clusterSortedIdList) + .filter((x) => Object.keys(activatedClusters).includes(x.id)); + + const [query, setQuery] = useState(() => { const now = dayjs(); return { @@ -77,7 +83,7 @@ export const AdminJobTable: React.FC = () => { userId: "", accountName: "", jobEndTime: [now.subtract(1, "week").startOf("day"), now.endOf("day")], - clusters: getSortedClusterValues(), + clusters: sortedClusters, }; }); const [form] = Form.useForm(); @@ -215,6 +221,7 @@ const JobInfoTable: React.FC = ({ const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); const [previewItem, setPreviewItem] = useState(undefined); @@ -272,7 +279,7 @@ const JobInfoTable: React.FC = ({ dataIndex="cluster" ellipsis title={t(pCommon("cluster"))} - render={(cluster) => getClusterName(cluster, languageId)} + render={(cluster) => getClusterName(cluster, languageId, publicConfigClusters)} /> dataIndex="partition" width="6.7%" ellipsis title={t(pCommon("partition"))} /> dataIndex="qos" width="6.7%" ellipsis title="QOS" /> diff --git a/apps/mis-web/src/pages/_app.tsx b/apps/mis-web/src/pages/_app.tsx index 6bc86da167..b1095fb575 100644 --- a/apps/mis-web/src/pages/_app.tsx +++ b/apps/mis-web/src/pages/_app.tsx @@ -14,14 +14,15 @@ import "nprogress/nprogress.css"; import "antd/dist/reset.css"; import { failEvent } from "@ddadaal/next-typed-api-routes-runtime/lib/client"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; import { UiExtensionStore } from "@scow/lib-web/build/extensions/UiExtensionStore"; import { DarkModeCookie, DarkModeProvider, getDarkModeCookieValue } from "@scow/lib-web/build/layouts/darkMode"; import { GlobalStyle } from "@scow/lib-web/build/layouts/globalStyle"; import { getHostname } from "@scow/lib-web/build/utils/getHostname"; import { useConstant } from "@scow/lib-web/build/utils/hooks"; import { isServer } from "@scow/lib-web/build/utils/isServer"; +import { formatActivatedClusters } from "@scow/lib-web/build/utils/misCommon/clustersActivation"; import { getCurrentLanguageId, getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; -// import { getInitialLanguage, getLanguageCookie } from "@scow/lib-web/build/utils/systemLanguage"; import { App as AntdApp } from "antd"; import type { AppContext, AppProps } from "next/app"; import App from "next/app"; @@ -39,10 +40,11 @@ import zh_cn from "src/i18n/zh_cn"; import { AntdConfigProvider } from "src/layouts/AntdConfigProvider"; import { BaseLayout } from "src/layouts/BaseLayout"; import { FloatButtons } from "src/layouts/FloatButtons"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { User, UserStore, } from "src/stores/UserStore"; +import { Cluster, getPublicConfigClusters } from "src/utils/cluster"; import { publicConfig, runtimeConfig } from "src/utils/config"; const languagesMap = { @@ -54,6 +56,7 @@ const languagesMap = { const FailEventHandler: React.FC = () => { const { message, modal } = AntdApp.useApp(); const userStore = useStore(UserStore); + const { publicConfigClusters, setActivatedClusters } = useStore(ClusterInfoStore); const languageId = useI18n().currentLanguage.id; const tArgs = useI18nTranslate(); @@ -78,7 +81,7 @@ const FailEventHandler: React.FC = () => { if (e.data?.code === "ADAPTER_CALL_ON_ONE_ERROR") { const clusterId = e.data.clusterErrorsArray[0].clusterId; const clusterName = clusterId ? - (publicConfig.CLUSTERS[clusterId]?.name ?? clusterId) : undefined; + (publicConfigClusters[clusterId]?.name ?? clusterId) : undefined; message.error(`${tArgs("page._app.adapterConnErrorContent", [getI18nConfigCurrentText(clusterName, languageId)])}(${ @@ -87,6 +90,30 @@ const FailEventHandler: React.FC = () => { return; } + if (e.data?.code === "NO_ACTIVATED_CLUSTERS") { + message.error(tArgs("page._app.noActivatedClusters")); + setActivatedClusters({}); + return; + } + + if (e.data?.code === "NOT_EXIST_IN_ACTIVATED_CLUSTERS") { + message.error(tArgs("page._app.notExistInActivatedClusters")); + + const currentActivatedClusterIds = e.data.currentActivatedClusterIds; + const newActivatedClusters: {[clusterId: string]: Cluster} = {}; + currentActivatedClusterIds.forEach((id: string) => { + if (publicConfigClusters[id]) { + newActivatedClusters[id] = publicConfigClusters[id]; + } + }); + setActivatedClusters(newActivatedClusters); + return; + } + + if (e.data?.code === "NO_CLUSTERS") { + message.error(tArgs("page._app.noClusters")); + return; + } message.error(`${tArgs("page._app.effectErrorMessage")}(${e.status}, ${e.data?.code}))`); @@ -110,6 +137,8 @@ interface ExtraProps { footerText: string; darkModeCookieValue: DarkModeCookie | undefined; initialLanguage: string; + clusterConfigs: { [clusterId: string]: ClusterConfigSchema; }; + initialActivatedClusters: {[clusterId: string]: Cluster}; } type Props = AppProps & { extra: ExtraProps }; @@ -123,9 +152,8 @@ function MyApp({ Component, pageProps, extra }: Props) { return store; }); - const defaultClusterStore = useConstant(() => { - const store = createStore(DefaultClusterStore, publicConfig.CLUSTERS[publicConfig.CLUSTER_SORTED_ID_LIST[0]]); - return store; + const clusterInfoStore = useConstant(() => { + return createStore(ClusterInfoStore, extra.clusterConfigs, extra.initialActivatedClusters); }); const uiExtensionStore = useConstant(() => createStore(UiExtensionStore, publicConfig.UI_EXTENSION)); @@ -157,7 +185,7 @@ function MyApp({ Component, pageProps, extra }: Props) { definitions: languagesMap[extra.initialLanguage], }} > - + @@ -186,6 +214,8 @@ MyApp.getInitialProps = async (appContext: AppContext) => { primaryColor: "", darkModeCookieValue: getDarkModeCookieValue(appContext.ctx.req), initialLanguage: "", + clusterConfigs: {}, + initialActivatedClusters: {}, }; // This is called on server on first load, and on client on every page transition @@ -209,6 +239,28 @@ MyApp.getInitialProps = async (appContext: AppContext) => { ...result, token: token, }; + + // get cluster configs from config file + const data = await api.getClusterConfigFiles({ query: { token } }) + .then((x) => x, () => ({ clusterConfigs: {} })); + + const clusterConfigs = data?.clusterConfigs; + if (clusterConfigs && Object.keys(clusterConfigs).length > 0) { + + extra.clusterConfigs = clusterConfigs; + const publicConfigClusters + = getPublicConfigClusters(clusterConfigs); + // get initial activated clusters + const clustersRuntimeInfo = + await api.getClustersRuntimeInfo({ query: { token } }).then((x) => x, () => undefined); + + const activatedClusters + = formatActivatedClusters({ + clustersRuntimeInfo: clustersRuntimeInfo?.results, + misConfigClusters: publicConfigClusters }); + extra.initialActivatedClusters = activatedClusters.misActivatedClusters ?? {}; + + } } } @@ -224,7 +276,6 @@ MyApp.getInitialProps = async (appContext: AppContext) => { } const appProps = await App.getInitialProps(appContext); - return { ...appProps, extra } as Props; }; diff --git a/apps/mis-web/src/pages/admin/jobBilling.tsx b/apps/mis-web/src/pages/admin/jobBilling.tsx index c42e70a7e0..22fdf2a9c7 100644 --- a/apps/mis-web/src/pages/admin/jobBilling.tsx +++ b/apps/mis-web/src/pages/admin/jobBilling.tsx @@ -16,6 +16,7 @@ import { NextPage } from "next"; import Router from "next/router"; import React, { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; import { FilterFormContainer } from "src/components/FilterFormContainer"; @@ -24,6 +25,7 @@ import { prefix, useI18nTranslateToString } from "src/i18n"; import { PlatformRole } from "src/models/User"; import { ManageJobBillingTable } from "src/pageComponents/job/ManageJobBillingTable"; import { PlatformOrTenantRadio } from "src/pageComponents/job/PlatformOrTenantRadio"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; const p = prefix("page.admin.jobBilling."); @@ -49,12 +51,19 @@ export const AdminJobBillingTablePage: NextPage = export const AdminJobBillingTable: React.FC<{ tenant?: string }> = ({ tenant }) => { const t = useI18nTranslateToString(); + + const { clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const currentActivatedClusterIds = Object.keys(activatedClusters); const { data, isLoading, reload } = useAsync({ promiseFn: useCallback(async () => { - return await api.getBillingItems({ query: { tenant, activeOnly: false } }); + return await api.getBillingItems({ + query: { tenant, activeOnly: false, currentActivatedClusterIds, clusterSortedIdList } }); }, [tenant]) }); return (
+ { currentActivatedClusterIds.length === 0 && +
{t("common.noAvailableClusters")}
+ }
diff --git a/apps/mis-web/src/pages/admin/resource/clusterManagement.tsx b/apps/mis-web/src/pages/admin/resource/clusterManagement.tsx new file mode 100644 index 0000000000..4f5ad78bae --- /dev/null +++ b/apps/mis-web/src/pages/admin/resource/clusterManagement.tsx @@ -0,0 +1,104 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { ClusterActivationStatus } from "@scow/config/build/type"; +import { RefreshLink, useRefreshToken } from "@scow/lib-web/build/utils/refreshToken"; +import { NextPage } from "next"; +import { useCallback } from "react"; +import { useAsync } from "react-async"; +import { useStore } from "simstate"; +import { api } from "src/apis"; +import { requireAuth } from "src/auth/requireAuth"; +import { PageTitle } from "src/components/PageTitle"; +import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; +import { ClusterConnectionStatus, Partition } from "src/models/cluster"; +import { PlatformRole } from "src/models/User"; +import { ClusterManagementTable } from "src/pageComponents/admin/ClusterManagementTable"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; +import { Head } from "src/utils/head"; + + +export interface CombinedClusterInfo { + clusterId: string, + schedulerName: string, + connectionStatus: ClusterConnectionStatus, + partitions: Partition[], + activationStatus: ClusterActivationStatus, + deactivationComment?: string, + operatorId?: string, + operatorName?: string, + updateTime: string, + hpcEnabled?: boolean, +} + +export const ClusterManagementPage: NextPage = + requireAuth((u) => u.platformRoles.includes(PlatformRole.PLATFORM_ADMIN))(() => { + + const t = useI18nTranslateToString(); + const languageId = useI18n().currentLanguage.id; + const p = prefix("page.admin.resourceManagement.clusterManagement."); + + const { publicConfigClusters, clusterSortedIdList, setActivatedClusters } = useStore(ClusterInfoStore); + + const promiseFn = useCallback(async () => { + const [connectionClustersData, dbClustersData] = await Promise.all([ + api.getClustersConnectionInfo({}), + api.getClustersRuntimeInfo({ query: {} }), + ]); + + const combinedClusterList: CombinedClusterInfo[] = []; + const currentActivatedClusters: {[clusterId: string]: Cluster} = {}; + // sort by cluster's priority + const sortedConnectionClustersData = connectionClustersData.results.sort((a, b) => { + const sortedIds = clusterSortedIdList; + return sortedIds.indexOf(a.clusterId) - sortedIds.indexOf(b.clusterId); + }); + sortedConnectionClustersData.forEach((cluster) => { + const currentCluster = dbClustersData.results.find((dbCluster) => dbCluster.clusterId === cluster.clusterId); + if (currentCluster) { + const combinedData = { + ...cluster, + ...currentCluster, + } as CombinedClusterInfo; + combinedClusterList.push(combinedData); + if (combinedData.activationStatus === ClusterActivationStatus.ACTIVATED) { + currentActivatedClusters[combinedData.clusterId] = publicConfigClusters[combinedData.clusterId]; + } + } + }); + setActivatedClusters(currentActivatedClusters); + return combinedClusterList; + + }, []); + + const [refreshToken, update] = useRefreshToken(); + + const { data, isLoading, reload } = useAsync({ promiseFn, watch: refreshToken }); + + return ( +
+ + + + + +
+ ); + + }); + +export default ClusterManagementPage; diff --git a/apps/mis-web/src/pages/api/admin/activateCluster.ts b/apps/mis-web/src/pages/api/admin/activateCluster.ts new file mode 100644 index 0000000000..cbb20097f0 --- /dev/null +++ b/apps/mis-web/src/pages/api/admin/activateCluster.ts @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { Status } from "@grpc/grpc-js/build/src/constants"; +import { ConfigServiceClient } from "@scow/protos/build/server/config"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { OperationResult, OperationType } from "src/models/operationLog"; +import { PlatformRole } from "src/models/User"; +import { callLog } from "src/server/operationLog"; +import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; +import { handlegRPCError, parseIp } from "src/utils/server"; + +export const ActivateClusterSchema = typeboxRouteSchema({ + method: "PUT", + + body: Type.Object({ + clusterId: Type.String(), + }), + + responses: { + // 如果当前集群无法连接或者已经上线了，那么executed为false + 200: Type.Object({ + executed: Type.Boolean(), + reason: Type.Optional(Type.String()), + }), + // 集群不存在 + 404: Type.Null(), + }, +}); + +export default /* #__PURE__*/route(ActivateClusterSchema, async (req, res) => { + const { clusterId } = req.body; + + const auth = authenticate((u) => u.platformRoles.includes(PlatformRole.PLATFORM_ADMIN)); + + const info = await auth(req, res); + + if (!info) { return; } + + + const client = getClient(ConfigServiceClient); + + const logInfo = { + operatorUserId: info.identityId, + operatorIp: parseIp(req) ?? "", + operationTypeName: OperationType.activateCluster, + operationTypePayload:{ + userId: info.identityId, clusterId, + }, + }; + + return await asyncClientCall(client, "activateCluster", { + clusterId, + operatorId: info.identityId, + }) + .then(async (reply) => { + await callLog(logInfo, OperationResult.SUCCESS); + return { 200: reply }; + }) + .catch(handlegRPCError({ + [Status.NOT_FOUND]: () => ({ 404: null }), + [Status.FAILED_PRECONDITION]: (e) => ({ 200: { executed: false, reason: e.details } }), + }, + async () => await callLog(logInfo, OperationResult.FAIL), + )); +}); diff --git a/apps/mis-web/src/pages/api/admin/deactivateCluster.ts b/apps/mis-web/src/pages/api/admin/deactivateCluster.ts new file mode 100644 index 0000000000..f9e4868789 --- /dev/null +++ b/apps/mis-web/src/pages/api/admin/deactivateCluster.ts @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { Status } from "@grpc/grpc-js/build/src/constants"; +import { ConfigServiceClient } from "@scow/protos/build/server/config"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { OperationResult, OperationType } from "src/models/operationLog"; +import { PlatformRole } from "src/models/User"; +import { callLog } from "src/server/operationLog"; +import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; +import { handlegRPCError, parseIp } from "src/utils/server"; + +export const DeactivateClusterSchema = typeboxRouteSchema({ + method: "PUT", + + body: Type.Object({ + clusterId: Type.String(), + deactivationComment: Type.Optional(Type.String()), + }), + + responses: { + // 如果集群已经下线了，那么executed为false + 200: Type.Object({ + executed: Type.Boolean(), + }), + // 集群不存在 + 404: Type.Null(), + }, +}); + +export default /* #__PURE__*/route(DeactivateClusterSchema, async (req, res) => { + const { clusterId, deactivationComment } = req.body; + + const auth = authenticate((u) => u.platformRoles.includes(PlatformRole.PLATFORM_ADMIN)); + + const info = await auth(req, res); + + if (!info) { return; } + + + const client = getClient(ConfigServiceClient); + + const logInfo = { + operatorUserId: info.identityId, + operatorIp: parseIp(req) ?? "", + operationTypeName: OperationType.deactivateCluster, + operationTypePayload:{ + userId: info.identityId, clusterId, + }, + }; + + return await asyncClientCall(client, "deactivateCluster", { + clusterId, + deactivationComment, + operatorId: info.identityId, + }) + .then(async (reply) => { + await callLog(logInfo, OperationResult.SUCCESS); + return { 200: reply }; + }) + .catch(handlegRPCError({ + [Status.NOT_FOUND]: () => ({ 404: null }), + }, + async () => await callLog(logInfo, OperationResult.FAIL), + )); +}); diff --git a/apps/mis-web/src/pages/api/admin/getClustersConnectionInfo.ts b/apps/mis-web/src/pages/api/admin/getClustersConnectionInfo.ts new file mode 100644 index 0000000000..9cbf6d718b --- /dev/null +++ b/apps/mis-web/src/pages/api/admin/getClustersConnectionInfo.ts @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { ConfigServiceClient } from "@scow/protos/build/common/config"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { ClusterConnectionInfo, ClusterConnectionInfoSchema, ClusterConnectionStatus } from "src/models/cluster"; +import { PlatformRole } from "src/models/User"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; +import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; + +export const GetClustersConnectionInfoSchema = typeboxRouteSchema({ + + method: "GET", + + responses: { + 200: Type.Object({ + results: Type.Array(ClusterConnectionInfoSchema), + }), + + }, +}); + +const auth = authenticate((info) => info.platformRoles.includes(PlatformRole.PLATFORM_ADMIN)); + +export default route(GetClustersConnectionInfoSchema, + async (req, res) => { + + const info = await auth(req, res); + if (!info) { + return; + } + + const configClusters = await getClusterConfigFiles(); + + const clustersConnectionResp: ClusterConnectionInfo[] = []; + const client = getClient(ConfigServiceClient); + + await Promise.allSettled(Object.keys(configClusters).map(async (cluster) => { + const reply = await asyncClientCall(client, "getClusterConfig", { cluster }) + .catch((e) => { + console.info("Cluster Connection Error ( Cluster ID : %s , Details: %s ) .", cluster, e); + clustersConnectionResp.push({ + clusterId: cluster, + connectionStatus: ClusterConnectionStatus.ERROR, + partitions: [], + }); + }); + + if (reply) { + clustersConnectionResp.push({ + clusterId: cluster, + connectionStatus: ClusterConnectionStatus.AVAILABLE, + schedulerName: reply.schedulerName, + partitions: reply.partitions, + }); + } + + })); + + return { + 200: { results: clustersConnectionResp }, + }; + }); diff --git a/apps/mis-web/src/pages/api/admin/getClustersRuntimeInfo.ts b/apps/mis-web/src/pages/api/admin/getClustersRuntimeInfo.ts new file mode 100644 index 0000000000..bf75123339 --- /dev/null +++ b/apps/mis-web/src/pages/api/admin/getClustersRuntimeInfo.ts @@ -0,0 +1,87 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { ClusterRuntimeInfo, ClusterRuntimeInfoSchema } from "@scow/config/build/type"; +import { ClusterRuntimeInfo_LastActivationOperation, ConfigServiceClient } from "@scow/protos/build/server/config"; +import { UserServiceClient } from "@scow/protos/build/server/user"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { validateToken } from "src/auth/token"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; +import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; + +export const GetClustersRuntimeInfoSchema = typeboxRouteSchema({ + + method: "GET", + + // only set the token query when firstly used in getInitialProps + query: Type.Object({ + token: Type.Optional(Type.String()), + }), + + responses: { + 200: Type.Object({ + results: Type.Array(ClusterRuntimeInfoSchema), + }), + + }, +}); + +const auth = authenticate(() => true); + +export default route(GetClustersRuntimeInfoSchema, + async (req, res) => { + + const { token } = req.query; + // when firstly used in getInitialProps, check the token + // when logged in, use auth() + const info = token ? await validateToken(token) : await auth(req, res); + if (!info) { return { 403: null }; } + + const client = getClient(ConfigServiceClient); + const result = await asyncClientCall(client, "getClustersRuntimeInfo", {}); + const operatorIds = Array.from(new Set(result.results.map((x) => { + const lastActivationOperation = x.lastActivationOperation as ClusterRuntimeInfo_LastActivationOperation; + return lastActivationOperation?.operatorId ?? undefined; + }))); + + const userIds = operatorIds.filter((id) => typeof id === "string" && id !== undefined && id !== null) as string[]; + + const userClient = getClient(UserServiceClient); + const { users } = await asyncClientCall(userClient, "getUsersByIds", { + userIds, + }); + + const userMap = new Map(users.map((x) => [x.userId, x.userName])); + + const clusterConfigs = await getClusterConfigFiles(); + + const clustersDatabaseInfo: ClusterRuntimeInfo[] = result.results.map((x) => { + const lastActivationOperation = x.lastActivationOperation as ClusterRuntimeInfo_LastActivationOperation; + return { + ...x, + operatorId: lastActivationOperation?.operatorId ?? "", + operatorName: lastActivationOperation?.operatorId ? userMap.get(lastActivationOperation?.operatorId) : "", + deactivationComment: lastActivationOperation?.deactivationComment ?? "", + hpcEnabled: clusterConfigs[x.clusterId]?.hpc?.enabled, + }; + }); + + return { + 200: { + results: clustersDatabaseInfo, + }, + }; + }); diff --git a/apps/mis-web/src/pages/api/admin/synchronize/syncBlockStatus.ts b/apps/mis-web/src/pages/api/admin/synchronize/syncBlockStatus.ts index 05f3427ab1..30b90afc77 100644 --- a/apps/mis-web/src/pages/api/admin/synchronize/syncBlockStatus.ts +++ b/apps/mis-web/src/pages/api/admin/synchronize/syncBlockStatus.ts @@ -10,13 +10,14 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { AdminServiceClient } from "@scow/protos/build/server/admin"; import { Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; import { PlatformRole } from "src/models/User"; import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; export const SyncBlockStatusSchema = typeboxRouteSchema({ method: "PUT", @@ -34,7 +35,7 @@ export const SyncBlockStatusSchema = typeboxRouteSchema({ }); const auth = authenticate((info) => info.platformRoles.includes(PlatformRole.PLATFORM_ADMIN)); -export default typeboxRoute(SyncBlockStatusSchema, +export default route(SyncBlockStatusSchema, async (req, res) => { const info = await auth(req, res); diff --git a/apps/mis-web/src/pages/api/clusterConfigsInfo.ts b/apps/mis-web/src/pages/api/clusterConfigsInfo.ts new file mode 100644 index 0000000000..a614239c55 --- /dev/null +++ b/apps/mis-web/src/pages/api/clusterConfigsInfo.ts @@ -0,0 +1,68 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { validateToken } from "src/auth/token"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; +import { route } from "src/utils/route"; + + +export const getClusterConfigFilesSchema = typeboxRouteSchema({ + method: "GET", + + // only set the query value when firstly used in getInitialProps + query: Type.Object({ + token: Type.Optional(Type.String()), + }), + + responses: { + + 200: Type.Object({ + clusterConfigs: Type.Record(Type.String(), ClusterConfigSchema) }), + + 403: Type.Null(), + }, +}); + +const auth = authenticate(() => true); + +export default route(getClusterConfigFilesSchema, + async (req, res) => { + + const { token } = req.query; + + // when firstly used in getInitialProps, check the token + // when logged in, use auth() + const info = token ? await validateToken(token) : await auth(req, res); + if (!info) { return { 403: null }; } + + const modifiedClusters: Record = await getClusterConfigFiles(); + + return { + 200: { clusterConfigs: modifiedClusters }, + }; + }); diff --git a/apps/mis-web/src/pages/api/file/exportOperationLog.ts b/apps/mis-web/src/pages/api/file/exportOperationLog.ts index a4084d0387..83a769d869 100644 --- a/apps/mis-web/src/pages/api/file/exportOperationLog.ts +++ b/apps/mis-web/src/pages/api/file/exportOperationLog.ts @@ -49,7 +49,7 @@ export const GetOperationLogFilter = Type.Object({ export type GetOperationLogFilter = Static; -export const GetOperationLogsSchema = typeboxRouteSchema({ +export const ExportOperationLogSchema = typeboxRouteSchema({ method: "GET", @@ -111,7 +111,7 @@ const getExportSource = ( } }; -export default typeboxRoute(GetOperationLogsSchema, async (req, res) => { +export default typeboxRoute(ExportOperationLogSchema, async (req, res) => { const auth = authenticate(() => true); diff --git a/apps/mis-web/src/pages/api/job/getAvailableBillingTable.ts b/apps/mis-web/src/pages/api/job/getAvailableBillingTable.ts index 4a9180aeb0..ff1af1f4a4 100644 --- a/apps/mis-web/src/pages/api/job/getAvailableBillingTable.ts +++ b/apps/mis-web/src/pages/api/job/getAvailableBillingTable.ts @@ -17,6 +17,7 @@ import { JobBillingItem } from "@scow/protos/build/server/job"; import { UserStatus } from "@scow/protos/build/server/user"; import { Static, Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; +import { Partition } from "src/models/cluster"; import { getBillingItems } from "src/pages/api/job/getBillingItems"; import { getClient } from "src/utils/client"; import { moneyToString } from "src/utils/money"; @@ -53,17 +54,6 @@ export const JobBillingTableItem = Type.Object({ }); export type JobBillingTableItem = Static; -export const Partition = Type.Object({ - name: Type.String(), - memMb: Type.Number(), - cores: Type.Number(), - gpus: Type.Number(), - nodes: Type.Number(), - qos: Type.Optional(Type.Array(Type.String())), - comment: Type.Optional(Type.String()), -}); -export type Partition = Static; - export const ClusterPartitions = Type.Object({ cluster: Type.String(), partitions: Type.Array(Partition), diff --git a/apps/mis-web/src/pages/api/job/getBillingItems.ts b/apps/mis-web/src/pages/api/job/getBillingItems.ts index 0508bfb074..93383a1637 100644 --- a/apps/mis-web/src/pages/api/job/getBillingItems.ts +++ b/apps/mis-web/src/pages/api/job/getBillingItems.ts @@ -21,7 +21,6 @@ import { authenticate } from "src/auth/server"; import { PlatformRole } from "src/models/User"; import { Money } from "src/models/UserSchemaModel"; import { getClient } from "src/utils/client"; -import { publicConfig } from "src/utils/config"; // Cannot use BillingItemType from pageComponents/job/ManageJobBillingTable export const BillingItemType = Type.Object({ @@ -55,6 +54,13 @@ export const GetBillingItemsSchema = typeboxRouteSchema({ */ activeOnly: Type.Boolean(), + /** + * currently activated cluster ids only + */ + currentActivatedClusterIds: Type.Optional(Type.Array(Type.String())), + + clusterSortedIdList: Type.Array(Type.String()), + }), responses: { @@ -116,7 +122,7 @@ const calculateNextId = (data?: JobBillingItem[], tenant?: string) => { export default /* #__PURE__*/typeboxRoute(GetBillingItemsSchema, async (req, res) => { - const { tenant, activeOnly } = req.query; + const { tenant, activeOnly, currentActivatedClusterIds, clusterSortedIdList } = req.query; if (tenant) { const auth = authenticate((u) => u.platformRoles.includes(PlatformRole.PLATFORM_ADMIN) || (u.tenant === tenant)); @@ -145,7 +151,10 @@ export default /* #__PURE__*/typeboxRoute(GetBillingItemsSchema, async (req, res const result = { activeItems: [] as BillingItemType[], historyItems: [] as BillingItemType[], nextId }; - for (const cluster of publicConfig.CLUSTER_SORTED_ID_LIST) { + const sortedIds = clusterSortedIdList.filter((id) => currentActivatedClusterIds?.includes(id)); + if (sortedIds.length === 0) { console.info("Cluster ops failed , error details: No available clusters"); } + + for (const cluster of sortedIds) { const client = getClient(ConfigServiceClient); diff --git a/apps/mis-web/src/pages/dashboard.tsx b/apps/mis-web/src/pages/dashboard.tsx index 7f2ae26fe6..e754bfa4b4 100644 --- a/apps/mis-web/src/pages/dashboard.tsx +++ b/apps/mis-web/src/pages/dashboard.tsx @@ -122,6 +122,7 @@ export const getServerSideProps: GetServerSideProps = async ({ req }) => return prev; }, {} as Record); + return { props: { accounts, diff --git a/apps/mis-web/src/pages/tenant/jobBillingTable.tsx b/apps/mis-web/src/pages/tenant/jobBillingTable.tsx index 383997f6f2..ad78615aad 100644 --- a/apps/mis-web/src/pages/tenant/jobBillingTable.tsx +++ b/apps/mis-web/src/pages/tenant/jobBillingTable.tsx @@ -13,12 +13,14 @@ import { NextPage } from "next"; import { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; import { PageTitle } from "src/components/PageTitle"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { TenantRole } from "src/models/User"; import { ManageJobBillingTable } from "src/pageComponents/job/ManageJobBillingTable"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; const p = prefix("page.tenant.jobBillingTable."); @@ -31,15 +33,21 @@ export const TenantAdminJobBillingTablePage: NextPage = requireAuth( const tenant = userStore.user.tenant; const t = useI18nTranslateToString(); + const { clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); + const currentActivatedClusterIds = Object.keys(activatedClusters); const { data, isLoading, reload } = useAsync({ promiseFn: useCallback(async () => { - return await api.getBillingItems({ query: { tenant: tenant, activeOnly: false } }); + return await api.getBillingItems({ + query: { tenant: tenant, activeOnly: false, currentActivatedClusterIds, clusterSortedIdList } }); }, [userStore.user]) }); return (
+ { currentActivatedClusterIds.length === 0 && +
{t("common.noAvailableClusters")}
+ }
); diff --git a/apps/mis-web/src/pages/tenant/storage.tsx b/apps/mis-web/src/pages/tenant/storage.tsx index f8c721f3c6..562a4580ca 100644 --- a/apps/mis-web/src/pages/tenant/storage.tsx +++ b/apps/mis-web/src/pages/tenant/storage.tsx @@ -19,12 +19,13 @@ import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; import { SingleClusterSelector } from "src/components/ClusterSelector"; import { DisabledA } from "src/components/DisabledA"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { PageTitle } from "src/components/PageTitle"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { TenantRole } from "src/models/User"; import type { ChangeStorageMode } from "src/pages/api/admin/changeStorage"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { Cluster } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; import { Head } from "src/utils/head"; const p = prefix("page.tenant.storage."); @@ -48,10 +49,15 @@ const StorageForm: React.FC = () => { const [loading, setLoading] = useState(false); - const [current, setCurent] = useState(undefined); + const [current, setCurrent] = useState(undefined); const [currentLoading, setCurrentLoading] = useState(false); - const defaultClusterStore = useStore(DefaultClusterStore); + const { activatedClusters, defaultCluster } = useStore(ClusterInfoStore); + if (!defaultCluster && Object.keys(activatedClusters).length === 0) { + return ; + } + + const currentDefaultCluster = defaultCluster ?? Object.values(activatedClusters)[0]; const t = useI18nTranslateToString(); @@ -66,7 +72,7 @@ const StorageForm: React.FC = () => { .httpError(400, () => { message.error(t(p("balanceChangeIllegal"))); }) .then(({ currentQuota }) => { message.success(t(p("editSuccess"))); - setCurent(currentQuota); + setCurrent(currentQuota); }) .finally(() => setLoading(false)); }; @@ -82,7 +88,7 @@ const StorageForm: React.FC = () => { await api.queryStorageQuota({ query: { cluster: cluster.id, userId } }) .httpError(404, () => { message.error(t(p("userNotFound"))); }) .then(({ currentQuota }) => { - setCurent(currentQuota); + setCurrent(currentQuota); }) .finally(() => setCurrentLoading(false)); }; @@ -107,7 +113,7 @@ const StorageForm: React.FC = () => { name="cluster" label={t("common.cluster")} rules={[{ required: true }]} - initialValue={defaultClusterStore.cluster} + initialValue={currentDefaultCluster} > diff --git a/apps/mis-web/src/pages/user/partitions.tsx b/apps/mis-web/src/pages/user/partitions.tsx index 15a7a5ee07..9211cbca16 100644 --- a/apps/mis-web/src/pages/user/partitions.tsx +++ b/apps/mis-web/src/pages/user/partitions.tsx @@ -23,9 +23,10 @@ import { checkCookie } from "src/auth/server"; import { JobBillingTable } from "src/components/JobBillingTable"; import { PageTitle } from "src/components/PageTitle"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { UserStore } from "src/stores/UserStore"; import { getSortedClusterValues } from "src/utils/cluster"; -import { publicConfig, runtimeConfig } from "src/utils/config"; +import { runtimeConfig } from "src/utils/config"; import { Head } from "src/utils/head"; import { styled } from "styled-components"; @@ -63,11 +64,14 @@ export const PartitionsPage: NextPage = requireAuth(() => true)((props: P const [completedRequestCount, setCompletedRequestCount] = useState(0); const [renderData, setRenderData] = useState<{ [cluster: string]: JobBillingTableItem[] }>({}); - const clusters = getSortedClusterValues(); + const { publicConfigClusters, clusterSortedIdList, activatedClusters } = useStore(ClusterInfoStore); - publicConfig.CLUSTER_SORTED_ID_LIST.forEach((clusterId) => { + const clusters = getSortedClusterValues(publicConfigClusters, clusterSortedIdList) + .filter((x) => Object.keys(activatedClusters).includes(x.id)); + const sortedIds = clusterSortedIdList.filter((id) => activatedClusters[id]); + sortedIds.forEach((clusterId) => { useAsync({ promiseFn: useCallback(async () => { - const cluster = publicConfig.CLUSTERS[clusterId]; + const cluster = activatedClusters[clusterId]; return api.getAvailableBillingTable({ query: { cluster: cluster.id, tenant: user?.tenant, userId: user?.identityId } }) .then((data) => { @@ -93,7 +97,13 @@ export const PartitionsPage: NextPage = requireAuth(() => true)((props: P > <> - ) : null + ) : ( + clusters.length === 0 ? ( + <> + {t("common.noAvailableClusters")} + + ) : null + ) }
> { + + const client = getClient(ConfigServiceClient); + const result = await asyncClientCall(client, "getClusterConfigFiles", {}); + + const modifiedClusters: Record = getClusterConfigsTypeFormat(result.clusterConfigs); + + return modifiedClusters; + +} diff --git a/apps/mis-web/src/stores/ClusterInfoStore.ts b/apps/mis-web/src/stores/ClusterInfoStore.ts new file mode 100644 index 0000000000..36161fc8fd --- /dev/null +++ b/apps/mis-web/src/stores/ClusterInfoStore.ts @@ -0,0 +1,69 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { ClusterConfigSchema } from "@scow/config/build/cluster"; +import { getSortedClusterIds } from "@scow/lib-web/build/utils/cluster"; +import { useEffect, useState } from "react"; +import { Cluster, getPublicConfigClusters } from "src/utils/cluster"; + +// export function ClusterInfoStore( +export function ClusterInfoStore( + clusterConfigs: {[clusterId: string]: ClusterConfigSchema}, + initialActivatedClusters: {[clusterId: string]: Cluster}, +) { + + const publicConfigClusters = getPublicConfigClusters(clusterConfigs); + + const clusterSortedIdList = getSortedClusterIds(clusterConfigs); + + const [activatedClusters, setActivatedClusters] + = useState<{[clusterId: string]: Cluster}>(initialActivatedClusters); + + const initialDefaultClusterId = clusterSortedIdList.find((x) => { + return Object.keys(initialActivatedClusters).find((c) => c === x); + }); + const initialDefaultCluster + = initialDefaultClusterId ? activatedClusters[initialDefaultClusterId] : undefined; + + const [ defaultCluster, setDefaultCluster ] = useState(initialDefaultCluster); + + useEffect(() => { + + // 可用集群不存在时 + if (Object.keys(activatedClusters).length === 0) { + setDefaultCluster(undefined); + } else { + + // 上一次记录的默认集群为undefined的情况，使用可用集群中的某一个集群作为新的默认集群 + if (!defaultCluster?.id) { + setDefaultCluster(Object.values(activatedClusters)[0]); + + // 上一次记录的默认集群已不在可用集群中的情况 + } else { + const currentDefaultExists = Object.keys(activatedClusters).find((x) => x === defaultCluster?.id); + if (!currentDefaultExists) { + setDefaultCluster(Object.values(activatedClusters)[0]); + } + } + } + + }, [activatedClusters]); + + return { + publicConfigClusters, + clusterSortedIdList, + activatedClusters, + setActivatedClusters, + defaultCluster, + setDefaultCluster, + }; +} diff --git a/apps/mis-web/src/utils/cluster.ts b/apps/mis-web/src/utils/cluster.ts index b5036f5b36..1950d810e1 100644 --- a/apps/mis-web/src/utils/cluster.ts +++ b/apps/mis-web/src/utils/cluster.ts @@ -10,14 +10,47 @@ * See the Mulan PSL v2 for more details. */ -import { Cluster, publicConfig } from "./config"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; +import { I18nStringType } from "@scow/config/build/i18n"; +import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; -export const getSortedClusterValues = (): Cluster[] => { +export type Cluster = { id: string; name: I18nStringType; } - const sortedClusters: Cluster[] = []; - publicConfig.CLUSTER_SORTED_ID_LIST.forEach((clusterId) => { - sortedClusters.push(publicConfig.CLUSTERS[clusterId]); - }); - - return sortedClusters; +export const getClusterName = ( + clusterId: string, + languageId: string, + publicConfigClusters: { [clusterId: string]: Cluster }, +) => { + return getI18nConfigCurrentText(publicConfigClusters[clusterId]?.name, languageId) || clusterId; }; + +export const getSortedClusterValues = + (publicConfigClusters: { [clusterId: string]: Cluster }, + clusterSortedIdList: string[], + ): Cluster[] => { + + const sortedClusters: Cluster[] = []; + clusterSortedIdList.forEach((clusterId) => { + sortedClusters.push(publicConfigClusters[clusterId]); + }); + + return sortedClusters; + }; + + +export const getPublicConfigClusters = + (configClusters: Record): + { [clusterId: string]: Cluster } => { + + const publicConfigClusters: { [clusterId: string]: Cluster; } = {}; + + Object.keys(configClusters).forEach((clusterId) => { + const cluster = { + id: clusterId, + name: configClusters[clusterId].displayName, + }; + publicConfigClusters[clusterId] = cluster; + }); + + return publicConfigClusters; + }; diff --git a/apps/mis-web/src/utils/config.ts b/apps/mis-web/src/utils/config.ts index 14597a27b9..05bd6124b7 100644 --- a/apps/mis-web/src/utils/config.ts +++ b/apps/mis-web/src/utils/config.ts @@ -11,7 +11,6 @@ */ import { AuditConfigSchema } from "@scow/config/build/audit"; -import type { ClusterConfigSchema } from "@scow/config/build/cluster"; import type { ClusterTextsConfigSchema } from "@scow/config/build/clusterTexts"; import { I18nStringType, SystemLanguageConfig } from "@scow/config/build/i18n"; import type { MisConfigSchema } from "@scow/config/build/mis"; @@ -30,8 +29,6 @@ export interface ServerRuntimeConfig { UI_CONFIG: UiConfigSchema | undefined; DEFAULT_PRIMARY_COLOR: string; - CLUSTERS_CONFIG: {[clusterId: string]: ClusterConfigSchema}; - CLUSTER_TEXTS_CONFIG: ClusterTextsConfigSchema; SCOW_API_AUTH_TOKEN?: string; @@ -47,10 +44,6 @@ export interface ServerRuntimeConfig { export interface PublicRuntimeConfig { BASE_PATH: string; - CLUSTERS: { [clusterId: string]: Cluster }; - - CLUSTER_SORTED_ID_LIST: string[]; - PREDEFINED_CHARGING_TYPES: string[]; CREATE_USER_CONFIG: { misConfig: MisConfigSchema["createUser"], @@ -118,7 +111,6 @@ export interface PublicRuntimeConfig { export const runtimeConfig: ServerRuntimeConfig = getConfig().serverRuntimeConfig; export const publicConfig: PublicRuntimeConfig = getConfig().publicRuntimeConfig; -export type Cluster = { id: string; name: I18nStringType; } export type NavLink = { text: string; url?: string; @@ -135,10 +127,6 @@ export type CustomAmountStrategy = { comment?: string | undefined; } -export const getClusterName = (clusterId: string, languageId: string) => { - return getI18nConfigCurrentText(publicConfig.CLUSTERS[clusterId]?.name, languageId) || clusterId; -}; - type ServerI18nConfigKeys = keyof typeof runtimeConfig.SERVER_I18N_CONFIG_TEXTS; // 获取ServerConfig中相关字符串配置的对应语言的字符串 export const getServerI18nConfigText = ( diff --git a/apps/mis-web/src/utils/route.ts b/apps/mis-web/src/utils/route.ts index 9d9fdf4eae..a4dbaf1f69 100644 --- a/apps/mis-web/src/utils/route.ts +++ b/apps/mis-web/src/utils/route.ts @@ -23,6 +23,7 @@ export const route: typeof typeboxRoute = (schema, handler) => { const response = handler(req, res); if (response instanceof Promise) { return response.catch((e) => { + if (!(e.metadata instanceof Metadata)) { throw e; } const SCOW_ERROR = e.metadata.get("IS_SCOW_ERROR"); @@ -32,9 +33,17 @@ export const route: typeof typeboxRoute = (schema, handler) => { // 如果包含集群详细错误信息 const clusterErrorsString = e.metadata.get("clusterErrors") ?? undefined; - const clusterErrorsArray = JSON.parse(clusterErrorsString) as ClusterErrorMetadata[]; + const clusterErrorsArray + = clusterErrorsString && clusterErrorsString.length > 0 ? + JSON.parse(clusterErrorsString) as ClusterErrorMetadata[] : undefined; + + // 如果包含当前在线集群的信息 + const currentActivatedClusterIdsStr = e.metadata.get("currentActivatedClusterIds") ?? undefined; + const currentActivatedClusterIds + = currentActivatedClusterIdsStr && currentActivatedClusterIdsStr.length > 0 ? + JSON.parse(currentActivatedClusterIdsStr) as string[] : undefined; - return { 500: { code, details, clusterErrorsArray } } as any; + return { 500: { code, details, currentActivatedClusterIds, clusterErrorsArray } } as any; }); } }); diff --git a/apps/portal-server/src/app.ts b/apps/portal-server/src/app.ts index 11491c2ae8..fc6e56b688 100644 --- a/apps/portal-server/src/app.ts +++ b/apps/portal-server/src/app.ts @@ -13,7 +13,7 @@ import { Server } from "@ddadaal/tsgrpc-server"; import { omitConfigSpec } from "@scow/lib-config"; import { readVersionFile } from "@scow/utils/build/version"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { config } from "src/config/env"; import { plugins } from "src/plugins"; import { appServiceServer } from "src/services/app"; @@ -54,7 +54,7 @@ export async function createServer() { if (process.env.NODE_ENV === "production") { await checkClustersRootUserLogin(server.logger); - await Promise.all(Object.entries(clusters).map(async ([id]) => { + await Promise.all(Object.entries(configClusters).map(async ([id]) => { await initShellFile(id, server.logger); })); await setupProxyGateway(server.logger); diff --git a/apps/portal-server/src/clusterops/app.ts b/apps/portal-server/src/clusterops/app.ts index ce03a75e66..3fedc73b48 100644 --- a/apps/portal-server/src/clusterops/app.ts +++ b/apps/portal-server/src/clusterops/app.ts @@ -24,7 +24,7 @@ import fs from "fs"; import { join } from "path"; import { quote } from "shell-quote"; import { AppOps, AppSession, SubmissionInfo } from "src/clusterops/api/app"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { portalConfig } from "src/config/portal"; import { getClusterAppConfigs, splitSbatchArgs } from "src/utils/app"; import { callOnOne } from "src/utils/clusters"; @@ -469,6 +469,7 @@ export const appOps = (cluster: string): AppOps => { if (displayId) { // the server is run at the compute node + const clusters = configClusters; // if proxyGateway configured, connect to compute node by proxyGateway and get ip of compute node const proxyGatewayConfig = clusters?.[cluster]?.proxyGateway; if (proxyGatewayConfig) { diff --git a/apps/portal-server/src/clusterops/index.ts b/apps/portal-server/src/clusterops/index.ts index f15398d998..6b8d03c044 100644 --- a/apps/portal-server/src/clusterops/index.ts +++ b/apps/portal-server/src/clusterops/index.ts @@ -13,7 +13,9 @@ import { ClusterOps } from "src/clusterops/api"; import { appOps } from "src/clusterops/app"; import { jobOps } from "src/clusterops/job"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; + +const clusters = configClusters; const opsForClusters = Object.entries(clusters).reduce((prev, [cluster]) => { prev[cluster] = { diff --git a/apps/portal-server/src/config/clusters.ts b/apps/portal-server/src/config/clusters.ts index dbe507a485..b6c15e0d3e 100644 --- a/apps/portal-server/src/config/clusters.ts +++ b/apps/portal-server/src/config/clusters.ts @@ -13,5 +13,6 @@ import { getClusterConfigs } from "@scow/config/build/cluster"; import { logger } from "src/utils/logger"; -export const clusters = getClusterConfigs(undefined, logger, ["hpc"]); + +export const configClusters = getClusterConfigs(undefined, logger, ["hpc"]); diff --git a/apps/portal-server/src/config/env.ts b/apps/portal-server/src/config/env.ts index 617d35016b..a68bd50e72 100644 --- a/apps/portal-server/src/config/env.ts +++ b/apps/portal-server/src/config/env.ts @@ -26,6 +26,9 @@ export const config = envConfig({ PORTAL_BASE_PATH: str({ desc: "门户系统base path", default: "/" }), + MIS_DEPLOYED: bool({ desc: "是否部署了管理系统", default: false }), + MIS_SERVER_URL: str({ desc: "如果部署了管理系统，管理系统后端服务的路径", default: "" }), + SSH_PRIVATE_KEY_PATH: str({ desc: "SSH私钥路径", default: join(homedir(), ".ssh", "id_rsa") }), SSH_PUBLIC_KEY_PATH: str({ desc: "SSH公钥路径", default: join(homedir(), ".ssh", "id_rsa.pub") }), diff --git a/apps/portal-server/src/services/app.ts b/apps/portal-server/src/services/app.ts index 78a9275ef5..aecfa4a6f7 100644 --- a/apps/portal-server/src/services/app.ts +++ b/apps/portal-server/src/services/app.ts @@ -14,19 +14,19 @@ import { plugin } from "@ddadaal/tsgrpc-server"; import { ServiceError } from "@grpc/grpc-js"; import { Status } from "@grpc/grpc-js/build/src/constants"; import { AppType } from "@scow/config/build/app"; -import { I18nStringType } from "@scow/config/build/i18n"; +import { getI18nSeverTypeFormat } from "@scow/lib-server"; import { AppCustomAttribute, appCustomAttribute_AttributeTypeFromJSON, AppServiceServer, AppServiceService, ConnectToAppResponse, - I18nStringProtoType, WebAppProps_ProxyType, } from "@scow/protos/build/portal/app"; import { DetailedError, ErrorInfo } from "@scow/rich-error-model"; import { getClusterOps } from "src/clusterops"; import { getClusterAppConfigs } from "src/utils/app"; +import { checkActivatedClusters } from "src/utils/clusters"; import { clusterNotFound } from "src/utils/errors"; const errorInfo = (reason: string) => @@ -38,6 +38,7 @@ export const appServiceServer = plugin((server) => { connectToApp: async ({ request, logger }) => { const { cluster, sessionId, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const apps = getClusterAppConfigs(cluster); @@ -96,6 +97,8 @@ export const appServiceServer = plugin((server) => { const { account, appId, appJobName, cluster, coreCount, nodeCount, gpuCount, memory, maxTime, proxyBasePath, partition, qos, userId, customAttributes } = request; + await checkActivatedClusters({ clusterIds: cluster }); + const apps = getClusterAppConfigs(cluster); const app = apps[appId]; @@ -179,7 +182,9 @@ export const appServiceServer = plugin((server) => { }, listAppSessions: async ({ request, logger }) => { + const { cluster, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const clusterops = getClusterOps(cluster); @@ -193,6 +198,8 @@ export const appServiceServer = plugin((server) => { getAppMetadata: async ({ request }) => { const { appId, cluster } = request; + await checkActivatedClusters({ clusterIds: cluster }); + const apps = getClusterAppConfigs(cluster); const app = apps[appId]; @@ -201,24 +208,6 @@ export const appServiceServer = plugin((server) => { } const attributes: AppCustomAttribute[] = []; - // config中的文本映射到protobuf中定义的grpc返回值的类型 - const getI18nSeverTypeFormat = (i18nConfig: I18nStringType): I18nStringProtoType | undefined => { - - if (!i18nConfig) return undefined; - - if (typeof i18nConfig === "string") { - return { value: { $case: "directString", directString: i18nConfig } }; - } else { - return { value: { $case: "i18nObject", i18nObject: { - i18n: { - default: i18nConfig.i18n.default, - en: i18nConfig.i18n.en, - zhCn: i18nConfig.i18n.zh_cn, - }, - } } }; - } - }; - if (app.attributes) { app.attributes.forEach((item) => { const attributeType = item.type.toUpperCase(); @@ -256,6 +245,7 @@ export const appServiceServer = plugin((server) => { listAvailableApps: async ({ request }) => { const { cluster } = request; + await checkActivatedClusters({ clusterIds: cluster }); const apps = getClusterAppConfigs(cluster); @@ -268,6 +258,8 @@ export const appServiceServer = plugin((server) => { getAppLastSubmission: async ({ request, logger }) => { const { userId, cluster, appId } = request; + await checkActivatedClusters({ clusterIds: cluster }); + const clusterops = getClusterOps(cluster); if (!clusterops) { throw clusterNotFound(cluster); } diff --git a/apps/portal-server/src/services/config.ts b/apps/portal-server/src/services/config.ts index 99e9bea3f1..1cf1de300d 100644 --- a/apps/portal-server/src/services/config.ts +++ b/apps/portal-server/src/services/config.ts @@ -11,19 +11,25 @@ */ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { ServiceError } from "@ddadaal/tsgrpc-common"; import { plugin } from "@ddadaal/tsgrpc-server"; -import { checkSchedulerApiVersion } from "@scow/lib-server"; +import { status } from "@grpc/grpc-js"; +import { getClusterConfigs } from "@scow/config/build/cluster"; +import { checkSchedulerApiVersion, convertClusterConfigsToServerProtoType, NO_CLUSTERS } from "@scow/lib-server"; +import { scowErrorMetadata } from "@scow/lib-server/build/error"; import { ConfigServiceServer, ConfigServiceService } from "@scow/protos/build/common/config"; import { ConfigServiceServer as runTimeConfigServiceServer, ConfigServiceService as runTimeConfigServiceService } from "@scow/protos/build/portal/config"; import { ApiVersion } from "@scow/utils/build/version"; -import { callOnOne } from "src/utils/clusters"; +import { callOnOne, checkActivatedClusters } from "src/utils/clusters"; export const staticConfigServiceServer = plugin((server) => { return server.addService(ConfigServiceService, { getClusterConfig: async ({ request, logger }) => { const { cluster } = request; + await checkActivatedClusters({ clusterIds: cluster }); + const reply = await callOnOne( cluster, logger, @@ -32,6 +38,25 @@ export const staticConfigServiceServer = plugin((server) => { return [reply]; }, + + getClusterConfigFiles: async ({ logger }) => { + + const clusterConfigs = getClusterConfigs(undefined, logger, ["hpc"]); + + const currentConfigClusterIds = Object.keys(clusterConfigs); + if (currentConfigClusterIds.length === 0) { + throw new ServiceError({ + code: status.INTERNAL, + details: "Unable to find cluster configuration files. Please contact the system administrator.", + metadata: scowErrorMetadata(NO_CLUSTERS), + }); + } + + const clusterConfigsProto = convertClusterConfigsToServerProtoType(clusterConfigs); + + return [{ clusterConfigs: clusterConfigsProto }]; + }, + }); }); diff --git a/apps/portal-server/src/services/dashboard.ts b/apps/portal-server/src/services/dashboard.ts index 5325b9b202..da61ab33dd 100644 --- a/apps/portal-server/src/services/dashboard.ts +++ b/apps/portal-server/src/services/dashboard.ts @@ -19,6 +19,7 @@ import path from "path"; const quickEntryPath = "/var/lib/scow/portal/quickEntries"; +// 在线集群单独处理 export const dashboardServiceServer = plugin((server) => { return server.addService(DashboardServiceService, { getQuickEntries:async ({ request, logger }) => { diff --git a/apps/portal-server/src/services/desktop.ts b/apps/portal-server/src/services/desktop.ts index 06d1bb0aa9..1b22b3fefb 100644 --- a/apps/portal-server/src/services/desktop.ts +++ b/apps/portal-server/src/services/desktop.ts @@ -16,7 +16,8 @@ import { Status } from "@grpc/grpc-js/build/src/constants"; import { getLoginNode } from "@scow/config/build/cluster"; import { executeAsUser } from "@scow/lib-ssh"; import { DesktopServiceServer, DesktopServiceService } from "@scow/protos/build/portal/desktop"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; +import { checkActivatedClusters } from "src/utils/clusters"; import { addDesktopToFile, ensureEnabled, @@ -35,6 +36,7 @@ export const desktopServiceServer = plugin((server) => { server.addService(DesktopServiceService, { createDesktop: async ({ request, logger }) => { const { cluster, loginNode: host, wm, userId, desktopName } = request; + await checkActivatedClusters({ clusterIds: cluster }); ensureEnabled(cluster); @@ -105,6 +107,7 @@ export const desktopServiceServer = plugin((server) => { killDesktop: async ({ request, logger }) => { const { cluster, loginNode: host, displayId, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); ensureEnabled(cluster); @@ -127,6 +130,7 @@ export const desktopServiceServer = plugin((server) => { connectToDesktop: async ({ request, logger }) => { const { cluster, loginNode: host, displayId, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); ensureEnabled(cluster); @@ -144,6 +148,7 @@ export const desktopServiceServer = plugin((server) => { listUserDesktops: async ({ request, logger }) => { const { cluster, loginNode: host, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); ensureEnabled(cluster); @@ -154,6 +159,7 @@ export const desktopServiceServer = plugin((server) => { return [{ userDesktops: [userDesktops]}]; } + const clusters = configClusters; const loginNodes = clusters[cluster]?.loginNodes?.map(getLoginNode); if (!loginNodes) { throw clusterNotFound(cluster); @@ -169,6 +175,7 @@ export const desktopServiceServer = plugin((server) => { listAvailableWms: async ({ request }) => { const { cluster } = request; + await checkActivatedClusters({ clusterIds: cluster }); ensureEnabled(cluster); diff --git a/apps/portal-server/src/services/file.ts b/apps/portal-server/src/services/file.ts index 287c7e0225..0adbfe8654 100644 --- a/apps/portal-server/src/services/file.ts +++ b/apps/portal-server/src/services/file.ts @@ -18,8 +18,9 @@ import { loggedExec, sftpAppendFile, sftpExists, sftpMkdir, sftpReaddir, import { FileInfo, FileInfo_FileType, FileServiceServer, FileServiceService, TransferInfo } from "@scow/protos/build/portal/file"; import { join } from "path"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { config } from "src/config/env"; +import { checkActivatedClusters } from "src/utils/clusters"; import { clusterNotFound } from "src/utils/errors"; import { pipeline } from "src/utils/pipeline"; import { getClusterLoginNode, getClusterTransferNode, sshConnect, tryGetClusterTransferNode } from "src/utils/ssh"; @@ -30,6 +31,7 @@ export const fileServiceServer = plugin((server) => { server.addService(FileServiceService, { copy: async ({ request, logger }) => { const { userId, cluster, fromPath, toPath } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -51,6 +53,7 @@ export const fileServiceServer = plugin((server) => { createFile: async ({ request, logger }) => { const { userId, cluster, path } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -72,6 +75,7 @@ export const fileServiceServer = plugin((server) => { deleteDirectory: async ({ request, logger }) => { const { userId, cluster, path } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -88,6 +92,7 @@ export const fileServiceServer = plugin((server) => { deleteFile: async ({ request, logger }) => { const { userId, cluster, path } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -105,6 +110,7 @@ export const fileServiceServer = plugin((server) => { getHomeDirectory: async ({ request, logger }) => { const { cluster, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -112,15 +118,14 @@ export const fileServiceServer = plugin((server) => { return await sshConnect(host, userId, logger, async (ssh) => { const sftp = await ssh.requestSFTP(); - const path = await sftpRealPath(sftp)("."); - return [{ path }]; }); }, makeDirectory: async ({ request, logger }) => { const { userId, cluster, path } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -143,6 +148,7 @@ export const fileServiceServer = plugin((server) => { move: async ({ request, logger }) => { const { userId, cluster, fromPath, toPath } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -161,6 +167,7 @@ export const fileServiceServer = plugin((server) => { readDirectory: async ({ request, logger }) => { const { userId, cluster, path, updateAccessTime } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -228,6 +235,7 @@ export const fileServiceServer = plugin((server) => { download: async (call) => { const { logger, request: { cluster, path, userId } } = call; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -283,6 +291,8 @@ export const fileServiceServer = plugin((server) => { const logger = call.logger.child({ upload: { userId, path, cluster, host } }); + await checkActivatedClusters({ clusterIds: cluster }); + logger.info("Upload file started"); return await sshConnect(host, userId, logger, async (ssh) => { @@ -355,6 +365,7 @@ export const fileServiceServer = plugin((server) => { getFileMetadata: async ({ request, logger }) => { const { userId, cluster, path } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -376,6 +387,7 @@ export const fileServiceServer = plugin((server) => { exists: async ({ request, logger }) => { const { userId, cluster, path } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); @@ -391,6 +403,8 @@ export const fileServiceServer = plugin((server) => { startFileTransfer: async ({ request, logger }) => { const { fromCluster, toCluster, userId, fromPath, toPath } = request; + await checkActivatedClusters({ clusterIds: [fromCluster, toCluster]}); + const fromTransferNodeAddress = getClusterTransferNode(fromCluster).address; const { host: toTransferNodeHost, @@ -430,6 +444,7 @@ export const fileServiceServer = plugin((server) => { queryFileTransfer: async ({ request, logger }) => { const { cluster, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const transferNodeAddress = getClusterTransferNode(cluster).address; @@ -459,6 +474,7 @@ export const fileServiceServer = plugin((server) => { const transferInfos: TransferInfo[] = []; // 根据host确定clusterId + const clusters = configClusters; transferInfosJsons.forEach((info) => { let toCluster = info.recvAddress; for (const key in clusters) { @@ -513,6 +529,8 @@ export const fileServiceServer = plugin((server) => { terminateFileTransfer: async ({ request, logger }) => { const { fromCluster, toCluster, userId, fromPath } = request; + await checkActivatedClusters({ clusterIds: [fromCluster, toCluster]}); + const fromTransferNodeAddress = getClusterTransferNode(fromCluster).address; const toTransferNodeHost = getClusterTransferNode(toCluster).host; @@ -542,6 +560,7 @@ export const fileServiceServer = plugin((server) => { checkTransferKey: async ({ request, logger }) => { const { fromCluster, toCluster, userId } = request; + await checkActivatedClusters({ clusterIds: [fromCluster, toCluster]}); const fromTransferNodeAddress = getClusterTransferNode(fromCluster).address; diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index 09091ee75b..50f7be8311 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -23,7 +23,7 @@ import { ApiVersion } from "@scow/utils/build/version"; import path, { join } from "path"; import { getClusterOps } from "src/clusterops"; import { JobTemplate } from "src/clusterops/api/job"; -import { callOnOne } from "src/utils/clusters"; +import { callOnOne, checkActivatedClusters } from "src/utils/clusters"; import { clusterNotFound } from "src/utils/errors"; import { getClusterLoginNode, sshConnect } from "src/utils/ssh"; @@ -34,6 +34,7 @@ export const jobServiceServer = plugin((server) => { cancelJob: async ({ request, logger }) => { const { cluster, jobId, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); await callOnOne( cluster, @@ -49,6 +50,7 @@ export const jobServiceServer = plugin((server) => { listAccounts: async ({ request, logger }) => { const { cluster, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const reply = await callOnOne( cluster, @@ -63,6 +65,7 @@ export const jobServiceServer = plugin((server) => { getJobTemplate: async ({ request, logger }) => { const { cluster, templateId, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const clusterops = getClusterOps(cluster); @@ -79,6 +82,7 @@ export const jobServiceServer = plugin((server) => { listJobTemplates: async ({ request, logger }) => { const { cluster, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const clusterops = getClusterOps(cluster); @@ -94,6 +98,8 @@ export const jobServiceServer = plugin((server) => { deleteJobTemplate: async ({ request, logger }) => { const { cluster, templateId, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); + const clusterops = getClusterOps(cluster); if (!clusterops) { throw clusterNotFound(cluster); } @@ -107,6 +113,8 @@ export const jobServiceServer = plugin((server) => { renameJobTemplate: async ({ request, logger }) => { const { cluster, templateId, userId, jobName } = request; + await checkActivatedClusters({ clusterIds: cluster }); + const clusterops = getClusterOps(cluster); if (!clusterops) { throw clusterNotFound(cluster); } @@ -121,6 +129,7 @@ export const jobServiceServer = plugin((server) => { listRunningJobs: async ({ request, logger }) => { const { cluster, userId } = request; + await checkActivatedClusters({ clusterIds: cluster }); const reply = await callOnOne( cluster, @@ -140,6 +149,7 @@ export const jobServiceServer = plugin((server) => { listAllJobs: async ({ request, logger }) => { const { cluster, userId, endTime, startTime } = request; + await checkActivatedClusters({ clusterIds: cluster }); const reply = await callOnOne( cluster, @@ -165,6 +175,7 @@ export const jobServiceServer = plugin((server) => { const { cluster, command, jobName, coreCount, gpuCount, maxTime, saveAsTemplate, userId, nodeCount, partition, qos, account, comment, workingDirectory, output , errorOutput, memory, scriptOutput } = request; + await checkActivatedClusters({ clusterIds: cluster }); // make sure working directory exists const host = getClusterLoginNode(cluster); @@ -239,6 +250,7 @@ export const jobServiceServer = plugin((server) => { submitFileAsJob: async ({ request, logger }) => { const { cluster, userId, filePath } = request; + await checkActivatedClusters({ clusterIds: cluster }); const host = getClusterLoginNode(cluster); if (!host) { throw clusterNotFound(cluster); } diff --git a/apps/portal-server/src/services/shell.ts b/apps/portal-server/src/services/shell.ts index f2fa447986..335ecdc2eb 100644 --- a/apps/portal-server/src/services/shell.ts +++ b/apps/portal-server/src/services/shell.ts @@ -15,6 +15,7 @@ import { plugin } from "@ddadaal/tsgrpc-server"; import { ServiceError, status } from "@grpc/grpc-js"; import { ShellServiceServer, ShellServiceService } from "@scow/protos/build/portal/shell"; import { quote } from "shell-quote"; +import { checkActivatedClusters } from "src/utils/clusters"; import { clusterNotFound } from "src/utils/errors"; import { pipeline } from "src/utils/pipeline"; import { sshConnect } from "src/utils/ssh"; @@ -40,6 +41,7 @@ export const shellServiceServer = plugin((server) => { logger.info("Received shell connection"); const { cluster, loginNode, userId, rows, cols, path } = connect; + await checkActivatedClusters({ clusterIds: cluster }); if (!loginNode) { throw clusterNotFound(cluster); } diff --git a/apps/portal-server/src/utils/clusters.ts b/apps/portal-server/src/utils/clusters.ts index f1a26a2588..17393278d9 100644 --- a/apps/portal-server/src/utils/clusters.ts +++ b/apps/portal-server/src/utils/clusters.ts @@ -13,11 +13,17 @@ import { ServiceError } from "@ddadaal/tsgrpc-common"; import { status } from "@grpc/grpc-js"; import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter"; -import { clusters } from "src/config/clusters"; +import { scowErrorMetadata } from "@scow/lib-server/build/error"; +import { libCheckActivatedClusters, + libGetCurrentActivatedClusters } from "@scow/lib-server/build/misCommon/clustersActivation"; +import { configClusters } from "src/config/clusters"; +import { commonConfig } from "src/config/common"; +import { config } from "src/config/env"; +import { logger as pinoLogger } from "src/utils/logger"; import { Logger } from "ts-log"; -import { scowErrorMetadata } from "./error"; +const clusters = configClusters; const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster, c]) => { const client = getSchedulerAdapterClient(c.adapterUrl); prev[cluster] = client; @@ -38,6 +44,9 @@ type CallOnOne = ( export const ADAPTER_CALL_ON_ONE_ERROR = "ADAPTER_CALL_ON_ONE_ERROR"; export const callOnOne: CallOnOne = async (cluster, logger, call) => { + + await checkActivatedClusters({ clusterIds: cluster }); + const client = getAdapterClient(cluster); if (!client) { @@ -71,3 +80,23 @@ export const callOnOne: CallOnOne = async (cluster, logger, call) => { }); }; + +export const checkActivatedClusters += async ( + { clusterIds }: {clusterIds: string[] | string}, +) => { + + if (!config.MIS_DEPLOYED) { + return; + } + + const activatedClusters = await libGetCurrentActivatedClusters( + pinoLogger, + configClusters, + config.MIS_SERVER_URL, + commonConfig.scowApi?.auth?.token); + + return libCheckActivatedClusters({ clusterIds, activatedClusters, logger: pinoLogger }); + +}; + diff --git a/apps/portal-server/src/utils/proxy.ts b/apps/portal-server/src/utils/proxy.ts index 296ee311cc..5bc10eed42 100644 --- a/apps/portal-server/src/utils/proxy.ts +++ b/apps/portal-server/src/utils/proxy.ts @@ -12,7 +12,7 @@ import { loggedExec, sftpWriteFile } from "@scow/lib-ssh"; import { dirname } from "path"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { config } from "src/config/env"; import { sshConnect } from "src/utils/ssh"; import { Logger } from "ts-log"; @@ -22,9 +22,9 @@ export const setupProxyGateway = async (logger: Logger) => { let portalBasePath = config.PORTAL_BASE_PATH; if (!portalBasePath.endsWith("/")) { portalBasePath += "/"; } - for (const id of Object.keys(clusters)) { + for (const id of Object.keys(configClusters)) { - const proxyGatewayConfig = clusters[id].proxyGateway; + const proxyGatewayConfig = configClusters[id].proxyGateway; if (!proxyGatewayConfig?.autoSetupNginx) { continue; } @@ -85,7 +85,7 @@ export const parseIp = (stdout: string): string => { export const getIpFromProxyGateway = async (clusterId: string, hostName: string, logger: Logger): Promise => { - const proxyGatewayConfig = clusters?.[clusterId]?.proxyGateway; + const proxyGatewayConfig = configClusters?.[clusterId]?.proxyGateway; if (!proxyGatewayConfig) return ""; const url = new URL(proxyGatewayConfig.url); diff --git a/apps/portal-server/src/utils/shell.ts b/apps/portal-server/src/utils/shell.ts index 866f6c8009..dfc7b41417 100644 --- a/apps/portal-server/src/utils/shell.ts +++ b/apps/portal-server/src/utils/shell.ts @@ -11,7 +11,7 @@ */ import { sftpChmod } from "@scow/lib-ssh"; -import { getClusterLoginNode, sshConnect } from "src/utils/ssh"; +import { getConfigClusterLoginNode, sshConnect } from "src/utils/ssh"; import { Logger } from "ts-log"; @@ -22,7 +22,7 @@ const SHELL_FILE_REMOTE = "/etc/profile.d/scow-shell-file.sh"; export async function initShellFile(cluster: string, logger: Logger) { - const host = getClusterLoginNode(cluster); + const host = getConfigClusterLoginNode(cluster); if (!host) { throw new Error(`Cluster ${cluster} has no login node`); } return await sshConnect(host, "root", logger, async (ssh) => { diff --git a/apps/portal-server/src/utils/ssh.ts b/apps/portal-server/src/utils/ssh.ts index 7c66c22911..5cd734f334 100644 --- a/apps/portal-server/src/utils/ssh.ts +++ b/apps/portal-server/src/utils/ssh.ts @@ -13,11 +13,11 @@ import { ServiceError } from "@ddadaal/tsgrpc-common"; import { status } from "@grpc/grpc-js"; import { getLoginNode } from "@scow/config/build/cluster"; +import { scowErrorMetadata } from "@scow/lib-server/build/error"; import { SftpError, sshConnect as libConnect, SshConnectError, testRootUserSshLogin } from "@scow/lib-ssh"; import { NodeSSH } from "node-ssh"; -import { clusters } from "src/config/clusters"; +import { configClusters } from "src/config/clusters"; import { rootKeyPair } from "src/config/env"; -import { scowErrorMetadata } from "src/utils/error"; import { Logger } from "ts-log"; import { clusterNotFound, loginNodeNotFound, transferNodeNotFound, transferNotEnabled } from "./errors"; @@ -28,14 +28,21 @@ interface NodeNetInfo { port: number, } +// 获取配置文件集群中各节点信息 +export function getConfigClusterLoginNode(cluster: string): string | undefined { + const loginNode = getLoginNode(configClusters[cluster]?.loginNodes?.[0]); + return loginNode?.address; +} + +// TODO: 不要？在线集群节点信息 export function getClusterLoginNode(cluster: string): string | undefined { - const loginNode = getLoginNode(clusters[cluster]?.loginNodes?.[0]); + const loginNode = getLoginNode(configClusters[cluster]?.loginNodes?.[0]); return loginNode?.address; } export function getClusterTransferNode(cluster: string): NodeNetInfo { - const enabled = clusters[cluster]?.crossClusterFileTransfer?.enabled; - const transferNode = clusters[cluster]?.crossClusterFileTransfer?.transferNode; + const enabled = configClusters[cluster]?.crossClusterFileTransfer?.enabled; + const transferNode = configClusters[cluster]?.crossClusterFileTransfer?.transferNode; if (!enabled) { throw transferNotEnabled(cluster); } @@ -55,8 +62,8 @@ export function getClusterTransferNode(cluster: string): NodeNetInfo { } export function tryGetClusterTransferNode(cluster: string): NodeNetInfo | undefined { - const enabled = clusters[cluster]?.crossClusterFileTransfer?.enabled; - const transferNode = clusters[cluster]?.crossClusterFileTransfer?.transferNode; + const enabled = configClusters[cluster]?.crossClusterFileTransfer?.enabled; + const transferNode = configClusters[cluster]?.crossClusterFileTransfer?.transferNode; if (!enabled) { return undefined; } @@ -113,7 +120,7 @@ export async function sshConnect( * Check whether all clusters can be logged in as root user */ export async function checkClustersRootUserLogin(logger: Logger) { - await Promise.all(Object.values(clusters).map(async ({ displayName, loginNodes }) => { + await Promise.all(Object.values(configClusters).map(async ({ displayName, loginNodes }) => { const node = getLoginNode(loginNodes[0]); logger.info("Checking if root can login to %s by login node %s", displayName, node.name); const error = await testRootUserSshLogin(node.address, rootKeyPair, console); @@ -130,7 +137,7 @@ export async function checkClustersRootUserLogin(logger: Logger) { * Check whether login node is in current cluster */ export async function checkLoginNodeInCluster(cluster: string, loginNode: string) { - const loginNodes = clusters[cluster]?.loginNodes.map(getLoginNode); + const loginNodes = configClusters[cluster]?.loginNodes.map(getLoginNode); if (!loginNodes) { throw clusterNotFound(cluster); } diff --git a/apps/portal-server/tests/file/utils.ts b/apps/portal-server/tests/file/utils.ts index b5fcbee46c..b01485f51a 100644 --- a/apps/portal-server/tests/file/utils.ts +++ b/apps/portal-server/tests/file/utils.ts @@ -12,8 +12,11 @@ import { ServiceError } from "@grpc/grpc-js"; import { I18nStringType } from "@scow/config/build/i18n"; +import { LoginNodesType } from "@scow/config/build/type"; import { sftpWriteFile, sshRawConnect, sshRmrf } from "@scow/lib-ssh"; -import { I18nStringProtoType, SubmissionInfo } from "@scow/protos/build/portal/app"; +import { ClusterConfigSchemaProto_LoginNodesProtoType } from "@scow/protos/build/common/config"; +import { I18nStringProtoType } from "@scow/protos/build/common/i18n"; +import { SubmissionInfo } from "@scow/protos/build/portal/app"; import { randomBytes } from "crypto"; import FormData from "form-data"; import { NodeSSH } from "node-ssh"; @@ -183,3 +186,21 @@ export const getI18nTypeFormat = (i18nProtoType: I18nStringProtoType | undefined } }; + + +// protobuf中定义的grpc返回值的loginNodes类型映射到前端loginNode +export const getLoginNodesTypeFormat = ( + protoType: ClusterConfigSchemaProto_LoginNodesProtoType | undefined): LoginNodesType => { + + if (!protoType?.value) return []; + if (protoType.value.$case === "loginNodeAddresses") { + return protoType.value.loginNodeAddresses.loginNodeAddressesValue; + } else { + const loginNodeConfigs = protoType.value.loginNodeConfigs; + return loginNodeConfigs.loginNodeConfigsValue.map((x) => ({ + name: getI18nTypeFormat(x.name), + address: x.address, + })); + } + +}; diff --git a/apps/portal-server/tests/utils/cluster.test.ts b/apps/portal-server/tests/utils/cluster.test.ts new file mode 100644 index 0000000000..d8ff9b3519 --- /dev/null +++ b/apps/portal-server/tests/utils/cluster.test.ts @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; +import { Server } from "@ddadaal/tsgrpc-server"; +import { credentials } from "@grpc/grpc-js"; +import { ClusterConfigSchema, getClusterConfigs } from "@scow/config/build/cluster"; +import { clusterConfigSchemaProto_K8sRuntimeToJSON, ConfigServiceClient } from "@scow/protos/build/common/config"; +import { createServer } from "src/app"; +import { logger } from "src/utils/logger"; +import { getI18nTypeFormat, getLoginNodesTypeFormat } from "tests/file/utils"; + +let server: Server; +let client: ConfigServiceClient; + +beforeEach(async () => { + + server = await createServer(); + + await server.start(); + + client = new ConfigServiceClient(server.serverAddress, credentials.createInsecure()); +}); + +afterEach(async () => { + await server.close(); +}); + +it("get cluster configs info", async () => { + + const clusterConfigsByReadingFiles = getClusterConfigs(undefined, logger, ["hpc"]); + + const reply = await asyncUnaryCall(client, "getClusterConfigFiles", { query: {} }); + + const clusterConfigsResp = reply.clusterConfigs; + + const modifiedClusters: Record = {}; + clusterConfigsResp.forEach((cluster) => { + const { clusterId, ... rest } = cluster; + const newCluster = { + ...rest, + displayName: getI18nTypeFormat(cluster.displayName), + loginNodes: !cluster.loginNodes ? [] : + getLoginNodesTypeFormat(cluster.loginNodes), + k8s: cluster.k8s ? { + k8sRuntime: clusterConfigSchemaProto_K8sRuntimeToJSON(cluster.k8s.runtime).toLowerCase(), + kubeconfig: cluster.k8s.kubeconfig, + } : undefined, + }; + modifiedClusters[cluster.clusterId] = newCluster as ClusterConfigSchema; + }); + + expect(modifiedClusters).toEqual(clusterConfigsByReadingFiles); +}); diff --git a/apps/portal-web/config.js b/apps/portal-web/config.js index 4f03ba3a4b..d1376b436f 100644 --- a/apps/portal-web/config.js +++ b/apps/portal-web/config.js @@ -12,7 +12,7 @@ // @ts-check -const { envConfig, str, bool, parseKeyValue } = require("@scow/lib-config"); +const { envConfig, str, bool } = require("@scow/lib-config"); const { join } = require("path"); const { homedir } = require("os"); const { PHASE_DEVELOPMENT_SERVER, @@ -22,8 +22,6 @@ const { readVersionFile } = require("@scow/utils/build/version"); const { getCapabilities } = require("@scow/lib-auth"); const { DEFAULT_PRIMARY_COLOR, getUiConfig } = require("@scow/config/build/ui"); const { getPortalConfig } = require("@scow/config/build/portal"); -const { getClusterConfigs, getLoginNode, getSortedClusters, - getSortedClusterIds } = require("@scow/config/build/cluster"); const { getCommonConfig, getSystemLanguageConfig } = require("@scow/config/build/common"); const { getAuditConfig } = require("@scow/config/build/audit"); @@ -42,35 +40,14 @@ async function queryCapabilities(authUrl, phase) { } } - -/** - * 当所有集群下都关闭桌面登录功能时，才关闭。 - * @param {Record} clusters - * @param {import("@scow/config/build/portal").PortalConfigSchema} portalConfig - * @returns {boolean} desktop login enable - */ -function getDesktopEnabled(clusters, portalConfig) { - const clusterDesktopEnabled = Object.keys(clusters).reduce( - ((pre, cur) => { - - const curClusterDesktopEnabled = clusters?.[cur]?.loginDesktop?.enabled !== undefined - ? !!clusters[cur]?.loginDesktop?.enabled - : portalConfig.loginDesktop.enabled; - - return pre || curClusterDesktopEnabled; - }), false, - ); - - return clusterDesktopEnabled; -} - const specs = { AUTH_EXTERNAL_URL: str({ desc: "认证系统的URL。如果和本系统域名相同，可以只写完整路径", default: "/auth" }), AUTH_INTERNAL_URL: str({ desc: "认证服务内网地址", default: "http://auth:5000" }), - LOGIN_NODES: str({ desc: "集群的登录节点。将会覆写配置文件。格式：集群ID=登录节点,集群ID=登录节点", default: "" }), + // 当前SCOW未使用覆写配置文件逻辑 + // LOGIN_NODES: str({ desc: "集群的登录节点。将会覆写配置文件。格式：集群ID=登录节点,集群ID=登录节点", default: "" }), SSH_PRIVATE_KEY_PATH: str({ desc: "SSH私钥路径", default: join(homedir(), ".ssh", "id_rsa") }), SSH_PUBLIC_KEY_PATH: str({ desc: "SSH公钥路径", default: join(homedir(), ".ssh", "id_rsa.pub") }), @@ -81,6 +58,7 @@ const specs = { MIS_DEPLOYED: bool({ desc: "是否部署了管理系统", default: false }), MIS_URL: str({ desc: "如果部署了管理系统，管理系统的URL。如果和本系统域名相同，可以只写完整的路径。将会覆盖配置文件。空字符串等价于未部署管理系统", default: "" }), + MIS_SERVER_URL: str({ desc: "如果部署了管理系统，管理系统后端的路径", default: "" }), AI_DEPLOYED: bool({ desc: "是否部署了AI系统", default: false }), AI_URL: str({ desc: "如果部署了AI系统，AI系统的URL。如果和本系统域名相同，可以只写完整路径。将会覆盖配置文件。空字符串等价于未部署AI系统", default: "" }), @@ -130,9 +108,7 @@ const buildRuntimeConfig = async (phase, basePath) => { const configPath = mockEnv ? join(__dirname, "config") : undefined; - const clusters = getClusterConfigs(configPath, console, ["hpc"]); - - Object.keys(clusters).map((id) => clusters[id].loginNodes = clusters[id].loginNodes.map(getLoginNode)); + // Object.keys(clusters).map((id) => clusters[id].loginNodes = clusters[id].loginNodes.map(getLoginNode)); const uiConfig = getUiConfig(configPath, console); const portalConfig = getPortalConfig(configPath, console); @@ -146,16 +122,16 @@ const buildRuntimeConfig = async (phase, basePath) => { const serverRuntimeConfig = { AUTH_EXTERNAL_URL: config.AUTH_EXTERNAL_URL, AUTH_INTERNAL_URL: config.AUTH_INTERNAL_URL, - CLUSTERS_CONFIG: clusters, PORTAL_CONFIG: portalConfig, DEFAULT_PRIMARY_COLOR, MOCK_USER_ID: config.MOCK_USER_ID, UI_CONFIG: uiConfig, - LOGIN_NODES: parseKeyValue(config.LOGIN_NODES), + // 当前SCOW未使用？ + // LOGIN_NODES: parseKeyValue(config.LOGIN_NODES), SERVER_URL: config.SERVER_URL, SUBMIT_JOB_WORKING_DIR: portalConfig.submitJobDefaultPwd, SCOW_API_AUTH_TOKEN: commonConfig.scowApi?.auth?.token, - AUDIT_CONFIG: config.AUDIT_DEPLOYED ? auditConfig : undefined, + AUDIT_CONFIG : config.AUDIT_DEPLOYED ? auditConfig : undefined, SERVER_I18N_CONFIG_TEXTS: { submitJopPromptText: portalConfig.submitJobPromptText, @@ -167,8 +143,6 @@ const buildRuntimeConfig = async (phase, basePath) => { // query auth capabilities to set optional auth features const capabilities = await queryCapabilities(config.AUTH_INTERNAL_URL, phase); - const enableLoginDesktop = getDesktopEnabled(clusters, portalConfig); - const systemLanguageConfig = getSystemLanguageConfig(getCommonConfig().systemLanguage); /** @@ -182,17 +156,14 @@ const buildRuntimeConfig = async (phase, basePath) => { ENABLE_JOB_MANAGEMENT: portalConfig.jobManagement, - ENABLE_LOGIN_DESKTOP: enableLoginDesktop, - ENABLE_APPS: portalConfig.apps, MIS_URL: config.MIS_DEPLOYED ? (config.MIS_URL || portalConfig.misUrl) : undefined, - AI_URL: config.AI_DEPLOYED ? (config.AI_URL || portalConfig.aiUrl) : undefined, + MIS_DEPLOYED: config.MIS_DEPLOYED, + MIS_SERVER_URL: config.MIS_DEPLOYED ? config.MIS_SERVER_URL : undefined, - CLUSTERS: getSortedClusters(clusters).map((cluster) => ({ id: cluster.id, name: cluster.displayName })), - - CLUSTER_SORTED_ID_LIST: getSortedClusterIds(clusters), + AI_URL: config.AI_DEPLOYED ? (config.AI_URL || portalConfig.aiUrl) : undefined, NOVNC_CLIENT_URL: config.NOVNC_CLIENT_URL, @@ -206,10 +177,6 @@ const buildRuntimeConfig = async (phase, basePath) => { FILE_PREVIEW_SIZE: portalConfig.file?.preview.limitSize, - CROSS_CLUSTER_FILE_TRANSFER_ENABLED: - Object.values(clusters).filter( - (cluster) => cluster.crossClusterFileTransfer?.enabled).length > 1, - PUBLIC_PATH: config.PUBLIC_PATH, NAV_LINKS: portalConfig.navLinks, diff --git a/apps/portal-web/env/.env.dev b/apps/portal-web/env/.env.dev index d147e06db4..cd904d5bb5 100644 --- a/apps/portal-web/env/.env.dev +++ b/apps/portal-web/env/.env.dev @@ -2,6 +2,7 @@ MIS_URL=/admin MOCK_USER_ID=test MIS_DEPLOYED=true MIS_URL=/mis +MIS_SEVER_URL=mis-server:5000 SERVER_URL=localhost:5000 AI_DEPLOYED=true AI_URL=/ai diff --git a/apps/portal-web/src/apis/api.mock.ts b/apps/portal-web/src/apis/api.mock.ts index d2bdd437a6..b12185a5d9 100644 --- a/apps/portal-web/src/apis/api.mock.ts +++ b/apps/portal-web/src/apis/api.mock.ts @@ -11,6 +11,7 @@ */ import { JsonFetchResultPromiseLike } from "@ddadaal/next-typed-api-routes-runtime/lib/client"; +import { ClusterActivationStatus } from "@scow/config/build/type"; import type { RunningJob } from "@scow/protos/build/common/job"; import { JobInfo } from "@scow/protos/build/portal/job"; import { api } from "src/apis/api"; @@ -273,5 +274,29 @@ export const mockApi: MockApi = { terminateFileTransfer: null, checkTransferKey: null, + getClusterConfigFiles: async () => ({ clusterConfigs: { + hpc01: { + displayName: "hpc01Name", + priority: 1, + adapterUrl: "0.0.0.0:0000", + proxyGateway: undefined, + loginNodes: [{ "address": "localhost:22222", "name": "login" }], + loginDesktop: undefined, + turboVncPath: undefined, + crossClusterFileTransfer: undefined, + hpc: { enabled: true }, + ai: { enabled: false }, + k8s: undefined, + }, + } }), + + getClustersRuntimeInfo: async () => ({ results: [{ + clusterId: "hpc01", + activationStatus: ClusterActivationStatus.ACTIVATED, + operatorId: undefined, + operatorName: undefined, + comment: "", + }]}), + }; diff --git a/apps/portal-web/src/apis/api.ts b/apps/portal-web/src/apis/api.ts index 0f685438c2..3ea8db7fa5 100644 --- a/apps/portal-web/src/apis/api.ts +++ b/apps/portal-web/src/apis/api.ts @@ -14,6 +14,8 @@ import { apiClient } from "src/apis/client"; import type { GetClusterInfoSchema } from "src/pages/api//cluster"; +import type { getClusterConfigFilesSchema } from "src/pages/api//getClusterConfigFiles"; +import type { GetClustersRuntimeInfoSchema } from "src/pages/api//getClustersRuntimeInfo"; import type { CheckAppConnectivitySchema } from "src/pages/api/app/checkConnectivity"; import type { ConnectToAppSchema } from "src/pages/api/app/connectToApp"; import type { CreateAppSessionSchema } from "src/pages/api/app/createAppSession"; @@ -100,6 +102,8 @@ export const api = { startFileTransfer: apiClient.fromTypeboxRoute("POST", "/api/file/startFileTransfer"), terminateFileTransfer: apiClient.fromTypeboxRoute("POST", "/api/file/terminateFileTransfer"), uploadFile: apiClient.fromTypeboxRoute("POST", "/api/file/upload"), + getClusterConfigFiles: apiClient.fromTypeboxRoute("GET", "/api//getClusterConfigFiles"), + getClustersRuntimeInfo: apiClient.fromTypeboxRoute("GET", "/api//getClustersRuntimeInfo"), cancelJob: apiClient.fromTypeboxRoute("DELETE", "/api/job/cancelJob"), deleteJobTemplate: apiClient.fromTypeboxRoute("DELETE", "/api/job/deleteJobTemplate"), getAccounts: apiClient.fromTypeboxRoute("GET", "/api/job/getAccounts"), diff --git a/apps/portal-web/src/components/ClusterSelector.tsx b/apps/portal-web/src/components/ClusterSelector.tsx index 01178d49c3..42dd880d19 100644 --- a/apps/portal-web/src/components/ClusterSelector.tsx +++ b/apps/portal-web/src/components/ClusterSelector.tsx @@ -14,8 +14,8 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLangua import { Select } from "antd"; import { useStore } from "simstate"; import { useI18n, useI18nTranslateToString } from "src/i18n"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { Cluster, publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; interface Props { value?: Cluster[]; @@ -27,6 +27,8 @@ export const ClusterSelector: React.FC = ({ value, onChange }) => { const languageId = useI18n().currentLanguage.id; const t = useI18nTranslateToString(); + const { currentClusters } = useStore(ClusterInfoStore); + return (
t(p("title"))} - tableLayout="fixed" - dataSource={(clusterInfo.map((x) => ({ clusterName:x.clusterName, info:{ ...x } })) as Array) - .concat(failedClusters)} - loading={isLoading} - pagination={false} - scroll={{ y:275 }} - rowClassName={(tableProps) => (tableProps.info?.id === selectId ? "rowBgColor" : "")} - onRow={(r) => { - return { - onClick() { - if (r.info?.id !== undefined) { - setSelectId(r.info?.id); - } - }, - }; - }} - > - - dataIndex="clusterName" - width="15%" - title={t(p("clusterName"))} - sorter={(a, b, sortOrder) => - compareWithUndefined(getI18nConfigCurrentText(a.clusterName, languageId), - getI18nConfigCurrentText(b.clusterName, languageId), sortOrder)} - render={(clusterName) => getI18nConfigCurrentText(clusterName, languageId)} - /> - - dataIndex="partitionName" - title={t(p("partitionName"))} - sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.partitionName, b.info?.partitionName, sortOrder)} - render={(_, r) => r.info?.partitionName ?? "-"} - /> - - dataIndex="nodeCount" - title={t(p("nodeCount"))} - sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.nodeCount, b.info?.nodeCount, sortOrder)} - render={(_, r) => r.info?.nodeCount ?? "-"} - /> - - dataIndex="usageRatePercentage" - title={t(p("usageRatePercentage"))} - sorter={(a, b, sortOrder) => - compareWithUndefined(a.info?.usageRatePercentage, b.info?.usageRatePercentage, sortOrder)} - render={(_, r) => ( - - {r.info?.usageRatePercentage ? `${r.info.usageRatePercentage}%` : "-"} - - )} - /> - - dataIndex="cpuUsage" - title={t(p("cpuUsage"))} - sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.cpuUsage, b.info?.cpuUsage, sortOrder)} - render={(_, r) => ( - - {r.info?.cpuUsage !== undefined ? Number(r.info?.cpuUsage).toFixed(2) + "%" : "-"} - - )} - /> - - dataIndex="gpuUsage" - title={t(p("gpuUsage"))} - sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.gpuUsage, b.info?.gpuUsage, sortOrder) } - render={(_, r) => ( - - {r.info?.gpuUsage !== undefined ? Number(r.info?.gpuUsage).toFixed(2) + "%" : "-"} - - )} - /> - - dataIndex="pendingJobCount" - title={t(p("pendingJobCount"))} - sorter={(a, b, sortOrder) => - compareWithUndefined(a.info?.pendingJobCount, b.info?.pendingJobCount, sortOrder)} - render={(_, r) => r.info?.pendingJobCount ?? "-" } - /> - - dataIndex="partitionStatus" - title={t(p("partitionStatus"))} - sorter={(a, b, sortOrder) => - compareWithUndefined(a.info?.partitionStatus, b.info?.partitionStatus, sortOrder)} - render={(_, r) => r.info?.partitionStatus === 0 ? - {t(p("notAvailable"))} : {t(p("available"))} - } - /> -
- - + (isLoading || clusterInfo.length > 0) ? ( + + t(p("title"))} + tableLayout="fixed" + dataSource={(clusterInfo.map((x) => ({ clusterName:x.clusterName, info:{ ...x } })) as Array) + .concat(failedClusters)} + loading={isLoading} + pagination={false} + scroll={{ y:275 }} + rowClassName={(tableProps) => (tableProps.info?.id === selectId ? "rowBgColor" : "")} + onRow={(r) => { + return { + onClick() { + if (r.info?.id !== undefined) { + setSelectId(r.info?.id); + } + }, + }; + }} + > + + dataIndex="clusterName" + width="15%" + title={t(p("clusterName"))} + sorter={(a, b, sortOrder) => + compareWithUndefined(getI18nConfigCurrentText(a.clusterName, languageId), + getI18nConfigCurrentText(b.clusterName, languageId), sortOrder)} + render={(clusterName) => getI18nConfigCurrentText(clusterName, languageId)} + /> + + dataIndex="partitionName" + title={t(p("partitionName"))} + sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.partitionName, b.info?.partitionName, sortOrder)} + render={(_, r) => r.info?.partitionName ?? "-"} + /> + + dataIndex="nodeCount" + title={t(p("nodeCount"))} + sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.nodeCount, b.info?.nodeCount, sortOrder)} + render={(_, r) => r.info?.nodeCount ?? "-"} + /> + + dataIndex="usageRatePercentage" + title={t(p("usageRatePercentage"))} + sorter={(a, b, sortOrder) => + compareWithUndefined(a.info?.usageRatePercentage, b.info?.usageRatePercentage, sortOrder)} + render={(_, r) => ( + + {r.info?.usageRatePercentage ? `${r.info.usageRatePercentage}%` : "-"} + + )} + /> + + dataIndex="cpuUsage" + title={t(p("cpuUsage"))} + sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.cpuUsage, b.info?.cpuUsage, sortOrder)} + render={(_, r) => ( + + {r.info?.cpuUsage !== undefined ? Number(r.info?.cpuUsage).toFixed(2) + "%" : "-"} + + )} + /> + + dataIndex="gpuUsage" + title={t(p("gpuUsage"))} + sorter={(a, b, sortOrder) => compareWithUndefined(a.info?.gpuUsage, b.info?.gpuUsage, sortOrder) } + render={(_, r) => ( + + {r.info?.gpuUsage !== undefined ? Number(r.info?.gpuUsage).toFixed(2) + "%" : "-"} + + )} + /> + + dataIndex="pendingJobCount" + title={t(p("pendingJobCount"))} + sorter={(a, b, sortOrder) => + compareWithUndefined(a.info?.pendingJobCount, b.info?.pendingJobCount, sortOrder)} + render={(_, r) => r.info?.pendingJobCount ?? "-" } + /> + + dataIndex="partitionStatus" + title={t(p("partitionStatus"))} + sorter={(a, b, sortOrder) => + compareWithUndefined(a.info?.partitionStatus, b.info?.partitionStatus, sortOrder)} + render={(_, r) => r.info?.partitionStatus === 0 ? + {t(p("notAvailable"))} : {t(p("available"))} + } + /> +
+ + + ) : ( + } + > + {t("pages.common.noAvailableClusters")} + + ) ); }; diff --git a/apps/portal-web/src/pageComponents/dashboard/QuickEntry.tsx b/apps/portal-web/src/pageComponents/dashboard/QuickEntry.tsx index 6bb6aaf7fd..d5820a14c9 100644 --- a/apps/portal-web/src/pageComponents/dashboard/QuickEntry.tsx +++ b/apps/portal-web/src/pageComponents/dashboard/QuickEntry.tsx @@ -19,7 +19,7 @@ import { Localized, prefix } from "src/i18n"; import { DashboardSection } from "src/pageComponents/dashboard/DashboardSection"; import { Sortable } from "src/pageComponents/dashboard/Sortable"; import { App } from "src/pages/api/app/listAvailableApps"; -import { Cluster, publicConfig } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; import { styled } from "styled-components"; const CardsContainer = styled.div` @@ -35,7 +35,8 @@ export interface AppWithCluster { } interface Props { - + currentClusters: Cluster[]; + publicConfigClusters: Cluster[]; } export const defaultEntry: Entry[] = [ @@ -86,16 +87,15 @@ export const defaultEntry: Entry[] = [ ]; const p = prefix("pageComp.dashboard.quickEntry."); -export const QuickEntry: React.FC = () => { +export const QuickEntry: React.FC = ({ currentClusters, publicConfigClusters }) => { + const { data, isLoading:getQuickEntriesLoading } = useAsync({ promiseFn: useCallback(async () => { return await api.getQuickEntries({}); }, []) }); - const clusters = publicConfig.CLUSTERS; - // apps包含在哪些集群上可以创建app const { data:apps, isLoading:getAppsLoading } = useAsync({ promiseFn: useCallback(async () => { - const appsInfo = await Promise.all(clusters.map((x) => { + const appsInfo = await Promise.all(currentClusters.map((x) => { return api.listAvailableApps({ query: { cluster: x.id } }); })); @@ -114,11 +114,11 @@ export const QuickEntry: React.FC = () => { appWithCluster[y.id].app.logoPath = y.logoPath; } - appWithCluster[y.id].clusters.push(clusters[idx]); + appWithCluster[y.id].clusters.push(currentClusters[idx]); }); }); return appWithCluster; - }, [clusters]) }); + }, [currentClusters]) }); const [isEditable, setIsEditable] = useState(false); const [isFinished, setIsFinished] = useState(false); @@ -151,6 +151,8 @@ export const QuickEntry: React.FC = () => { isFinished={isFinished} quickEntryArray={data?.quickEntries.length ? data?.quickEntries : defaultEntry } apps={apps ?? {}} + currentClusters={currentClusters} + publicConfigClusters={publicConfigClusters} > )} diff --git a/apps/portal-web/src/pageComponents/dashboard/SelectClusterModal.tsx b/apps/portal-web/src/pageComponents/dashboard/SelectClusterModal.tsx index 4521631ba0..f31b709c5d 100644 --- a/apps/portal-web/src/pageComponents/dashboard/SelectClusterModal.tsx +++ b/apps/portal-web/src/pageComponents/dashboard/SelectClusterModal.tsx @@ -17,7 +17,7 @@ import React, { useState } from "react"; import { useStore } from "simstate"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { LoginNodeStore } from "src/stores/LoginNodeStore"; -import { Cluster } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; import { EntryCase, IncompleteEntryInfo } from "./AddEntryModal"; diff --git a/apps/portal-web/src/pageComponents/dashboard/Sortable.tsx b/apps/portal-web/src/pageComponents/dashboard/Sortable.tsx index b65bced19d..d68e761da6 100644 --- a/apps/portal-web/src/pageComponents/dashboard/Sortable.tsx +++ b/apps/portal-web/src/pageComponents/dashboard/Sortable.tsx @@ -31,7 +31,9 @@ import { useRouter } from "next/router"; import { join } from "path"; import { FC, useCallback, useEffect, useMemo, useState } from "react"; import { api } from "src/apis"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; +import { Cluster } from "src/utils/cluster"; import { formatEntryId, getEntryBaseName, getEntryExtraInfo, getEntryIcon, getEntryLogoPath } from "src/utils/dashboard"; import { styled } from "styled-components"; @@ -52,7 +54,9 @@ interface Props { isEditable: boolean, isFinished: boolean, quickEntryArray: Entry[], - apps: AppWithCluster; + apps: AppWithCluster, + currentClusters: Cluster[], + publicConfigClusters: Cluster[], } const p = prefix("pageComp.dashboard.sortable."); @@ -70,7 +74,8 @@ const DeleteIconContainer = styled.div` } `; -export const Sortable: FC = ({ isEditable, isFinished, quickEntryArray, apps }) => { +export const Sortable: FC = ({ + isEditable, isFinished, quickEntryArray, apps, currentClusters, publicConfigClusters }) => { const t = useI18nTranslateToString(); const i18n = useI18n(); @@ -145,13 +150,20 @@ export const Sortable: FC = ({ isEditable, isFinished, quickEntryArray, a break; case "shell": - router.push(join("/shell", item.entry.shell.clusterId, item.entry.shell.loginNode)); + const savedShellClusterId = item.entry.shell.clusterId; + router.push(join("/shell", savedShellClusterId, item.entry.shell.loginNode)); + if (!currentClusters.some((x) => x.id === savedShellClusterId)) { + return ; + } break; - case "app": + const savedAppClusterId = item.entry.app.clusterId; router.push( - join("/apps", item.entry.app.clusterId, "/create", item.entry.app.appId), + join("/apps", savedAppClusterId, "/create", item.entry.app.appId), ); + if (!currentClusters.some((x) => x.id === savedAppClusterId)) { + return ; + } break; default: @@ -215,7 +227,7 @@ export const Sortable: FC = ({ isEditable, isFinished, quickEntryArray, a id={x.id} key={x.id} entryBaseName={getEntryBaseName(x, t)} - entryExtraInfo={getEntryExtraInfo(x, i18n.currentLanguage.id)} + entryExtraInfo={getEntryExtraInfo(x, i18n.currentLanguage.id, publicConfigClusters)} draggable={isEditable} icon={getEntryIcon(x)} logoPath={getEntryLogoPath(x, apps)} @@ -244,7 +256,7 @@ export const Sortable: FC = ({ isEditable, isFinished, quickEntryArray, a isDragging id={activeId.toString()} entryBaseName={getEntryBaseName(activeItem, t)} - entryExtraInfo={getEntryExtraInfo(activeItem, i18n.currentLanguage.id)} + entryExtraInfo={getEntryExtraInfo(activeItem, i18n.currentLanguage.id, publicConfigClusters)} draggable={isEditable} icon={getEntryIcon(activeItem)} logoPath={getEntryLogoPath(activeItem, apps)} @@ -257,6 +269,7 @@ export const Sortable: FC = ({ isEditable, isFinished, quickEntryArray, a onClose={() => { setAddEntryOpen(false); }} apps={apps} addItem={addItem} + clusters={currentClusters} > ); diff --git a/apps/portal-web/src/pageComponents/desktop/DesktopTable.tsx b/apps/portal-web/src/pageComponents/desktop/DesktopTable.tsx index de58de8087..e15641e435 100644 --- a/apps/portal-web/src/pageComponents/desktop/DesktopTable.tsx +++ b/apps/portal-web/src/pageComponents/desktop/DesktopTable.tsx @@ -21,14 +21,15 @@ import { useAsync } from "react-async"; import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { FilterFormContainer } from "src/components/FilterFormContainer"; import { ModalButton } from "src/components/ModalLink"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { DesktopTableActions } from "src/pageComponents/desktop/DesktopTableActions"; import { NewDesktopTableModal } from "src/pageComponents/desktop/NewDesktopTableModal"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { LoginNodeStore } from "src/stores/LoginNodeStore"; -import { Cluster, publicConfig } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; const NewDesktopTableModalButton = ModalButton(NewDesktopTableModal, { type: "primary", icon: }); @@ -52,20 +53,24 @@ export const DesktopTable: React.FC = ({ loginDesktopEnabledClusters }) = const t = useI18nTranslateToString(); - const { defaultCluster } = useStore(DefaultClusterStore); + const { currentClusters, defaultCluster } = useStore(ClusterInfoStore); - const { loginNodes } = useStore(LoginNodeStore); + if (!defaultCluster && currentClusters.length === 0) { + return ; + } + const { loginNodes } = useStore(LoginNodeStore); const clusterQuery = queryToString(router.query.cluster); const loginQuery = queryToString(router.query.loginNode); const [selectedLoginNodeAddress, setSelectedLoginNodeAddress] = useState(""); - // 如果默认集群没开启登录节点桌面功能，则取开启此功能的某一集群为默认集群。 - const enabledDefaultCluster = loginDesktopEnabledClusters.find((x) => x.id === defaultCluster.id) - ? defaultCluster - : loginDesktopEnabledClusters[0]; - const cluster = publicConfig.CLUSTERS.find((x) => x.id === clusterQuery) ?? enabledDefaultCluster; + // 如果默认集群(不存在时优先选取可用集群中某一集群作为当前默认集群)没开启登录节点桌面功能，则取开启此功能的某一集群为默认集群 + const currentDefaultCluster = defaultCluster ?? currentClusters[0]; + const enabledDefaultCluster = loginDesktopEnabledClusters.find((x) => x.id === currentDefaultCluster.id) + ? currentDefaultCluster + : loginDesktopEnabledClusters[0]; + const cluster = currentClusters.find((x) => x.id === clusterQuery) ?? enabledDefaultCluster; const loginNode = loginNodes[cluster.id].find((x) => x.address === loginQuery) ?? undefined; const { data, isLoading, reload } = useAsync({ diff --git a/apps/portal-web/src/pageComponents/desktop/DesktopTableActions.tsx b/apps/portal-web/src/pageComponents/desktop/DesktopTableActions.tsx index 98cab4d01e..911d575fdc 100644 --- a/apps/portal-web/src/pageComponents/desktop/DesktopTableActions.tsx +++ b/apps/portal-web/src/pageComponents/desktop/DesktopTableActions.tsx @@ -15,7 +15,7 @@ import React, { useState } from "react"; import { api } from "src/apis"; import { prefix, useI18nTranslateToString } from "src/i18n"; import type { DesktopItem } from "src/pageComponents/desktop/DesktopTable"; -import { Cluster } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; import { openDesktop } from "src/utils/vnc"; interface Props { diff --git a/apps/portal-web/src/pageComponents/desktop/NewDesktopTableModal.tsx b/apps/portal-web/src/pageComponents/desktop/NewDesktopTableModal.tsx index 1b0b85fa2f..d3d53058c6 100644 --- a/apps/portal-web/src/pageComponents/desktop/NewDesktopTableModal.tsx +++ b/apps/portal-web/src/pageComponents/desktop/NewDesktopTableModal.tsx @@ -16,7 +16,7 @@ import dayjs from "dayjs"; import React, { useEffect, useState } from "react"; import { api } from "src/apis"; import { prefix, useI18nTranslateToString } from "src/i18n"; -import { Cluster, LoginNode } from "src/utils/config"; +import { Cluster, LoginNode } from "src/utils/cluster"; import { openDesktop } from "src/utils/vnc"; export interface Props { diff --git a/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx b/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx index 190a268e7d..68e2e00517 100644 --- a/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx @@ -21,7 +21,7 @@ import { api } from "src/apis"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { SingleCrossClusterTransferSelector } from "src/pageComponents/filemanager/SingleCrossClusterTransferSelector"; import { FileInfo } from "src/pages/api/file/list"; -import { Cluster } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; import { FileInfoKey, fileInfoKey, fileTypeIcons, nodeModeToString, openPreviewLink, TopBar } from "src/utils/file"; import { formatSize } from "src/utils/format"; import { styled } from "styled-components"; diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index e0e8644eac..31d111c8a9 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -44,7 +44,8 @@ import { RenameModal } from "src/pageComponents/filemanager/RenameModal"; import { UploadModal } from "src/pageComponents/filemanager/UploadModal"; import { FileInfo } from "src/pages/api/file/list"; import { LoginNodeStore } from "src/stores/LoginNodeStore"; -import { Cluster, publicConfig } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; +import { publicConfig } from "src/utils/config"; import { convertToBytes } from "src/utils/format"; import { canPreviewWithEditor, isImage } from "src/utils/staticFiles"; import { styled } from "styled-components"; diff --git a/apps/portal-web/src/pageComponents/filemanager/SingleCrossClusterTransferSelector.tsx b/apps/portal-web/src/pageComponents/filemanager/SingleCrossClusterTransferSelector.tsx index 6d1f3bae33..460e89af49 100644 --- a/apps/portal-web/src/pageComponents/filemanager/SingleCrossClusterTransferSelector.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/SingleCrossClusterTransferSelector.tsx @@ -16,7 +16,7 @@ import React, { useCallback } from "react"; import { useAsync } from "react-async"; import { api } from "src/apis"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; -import { Cluster } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; interface SingleSelectionProps { diff --git a/apps/portal-web/src/pageComponents/filemanager/TransferInfoTable.tsx b/apps/portal-web/src/pageComponents/filemanager/TransferInfoTable.tsx index d33fb50618..306a56f2ec 100644 --- a/apps/portal-web/src/pageComponents/filemanager/TransferInfoTable.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/TransferInfoTable.tsx @@ -16,7 +16,7 @@ import { useCallback, useEffect } from "react"; import { useAsync } from "react-async"; import { api } from "src/apis"; import { prefix, useI18nTranslateToString } from "src/i18n"; -import { Cluster } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; interface TransferData { cluster: string; diff --git a/apps/portal-web/src/pageComponents/job/AllJobsTable.tsx b/apps/portal-web/src/pageComponents/job/AllJobsTable.tsx index 5bc8cc46e9..161772044c 100644 --- a/apps/portal-web/src/pageComponents/job/AllJobsTable.tsx +++ b/apps/portal-web/src/pageComponents/job/AllJobsTable.tsx @@ -24,10 +24,11 @@ import { useAsync } from "react-async"; import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { FilterFormContainer } from "src/components/FilterFormContainer"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { Cluster } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; interface FilterForm { time: [dayjs.Dayjs, dayjs.Dayjs]; @@ -43,14 +44,18 @@ export const AllJobQueryTable: React.FC = ({ userId, }) => { - const { defaultCluster } = useStore(DefaultClusterStore); + const { currentClusters, defaultCluster } = useStore(ClusterInfoStore); + + if (!defaultCluster && currentClusters.length === 0) { + return ; + } const [query, setQuery] = useState(() => { const now = dayjs(); return { time: [now.subtract(1, "week").startOf("day"), now.endOf("day")], jobId: undefined, - cluster: defaultCluster, + cluster: defaultCluster ?? currentClusters[0], }; }); diff --git a/apps/portal-web/src/pageComponents/job/FileSelectModal.tsx b/apps/portal-web/src/pageComponents/job/FileSelectModal.tsx index 019209adbf..714b9a6185 100644 --- a/apps/portal-web/src/pageComponents/job/FileSelectModal.tsx +++ b/apps/portal-web/src/pageComponents/job/FileSelectModal.tsx @@ -24,7 +24,7 @@ import { FileTable } from "src/pageComponents/filemanager/FileTable"; import { MkdirModal } from "src/pageComponents/filemanager/MkdirModal"; import { PathBar } from "src/pageComponents/filemanager/PathBar"; import { FileInfo } from "src/pages/api/file/list"; -import { Cluster } from "src/utils/config"; +import { Cluster } from "src/utils/cluster"; import { styled } from "styled-components"; diff --git a/apps/portal-web/src/pageComponents/job/JobTemplateTable.tsx b/apps/portal-web/src/pageComponents/job/JobTemplateTable.tsx index 3d941619d7..aed6e23ec5 100644 --- a/apps/portal-web/src/pageComponents/job/JobTemplateTable.tsx +++ b/apps/portal-web/src/pageComponents/job/JobTemplateTable.tsx @@ -20,10 +20,11 @@ import { useAsync } from "react-async"; import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { FilterFormContainer } from "src/components/FilterFormContainer"; import { prefix, useI18nTranslateToString } from "src/i18n"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import type { Cluster } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; interface Props {} @@ -47,11 +48,15 @@ const p = prefix("pageComp.job.jobTemplateModal."); export const JobTemplateTable: React.FC = () => { - const { defaultCluster } = useStore(DefaultClusterStore); + const { currentClusters, defaultCluster } = useStore(ClusterInfoStore); + + if (!defaultCluster && currentClusters.length === 0) { + return ; + } const [query, setQuery] = useState(() => { return { - cluster: defaultCluster, + cluster: defaultCluster ?? currentClusters[0], }; }); diff --git a/apps/portal-web/src/pageComponents/job/RunningJobDrawer.tsx b/apps/portal-web/src/pageComponents/job/RunningJobDrawer.tsx index 471c26836c..95d82426e9 100644 --- a/apps/portal-web/src/pageComponents/job/RunningJobDrawer.tsx +++ b/apps/portal-web/src/pageComponents/job/RunningJobDrawer.tsx @@ -12,9 +12,11 @@ import { formatDateTime } from "@scow/lib-web/build/utils/datetime"; import { Descriptions, Drawer } from "antd"; +import { useStore } from "simstate"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { RunningJobInfo } from "src/models/job"; -import { getClusterName } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { getClusterName } from "src/utils/cluster"; interface Props { open: boolean; @@ -31,6 +33,7 @@ export const RunningJobDrawer: React.FC = ({ const t = useI18nTranslateToString(); const languageId = useI18n().currentLanguage.id; + const { publicConfigClusters } = useStore(ClusterInfoStore); const drawerItems = [ [t(p("cluster")), "cluster", getClusterName], @@ -68,7 +71,8 @@ export const RunningJobDrawer: React.FC = ({ {/* 如果是集群项展示，则根据当前语言id获取集群名称 */} {format ? - (key === "cluster" ? getClusterName(item[key].id, languageId) : format(item[key], item)) + (key === "cluster" ? + getClusterName(item[key].id, languageId, publicConfigClusters) : format(item[key], item)) : item[key] as string} ))} diff --git a/apps/portal-web/src/pageComponents/job/RunningJobTable.tsx b/apps/portal-web/src/pageComponents/job/RunningJobTable.tsx index 4b40a1e13d..41fcb47f29 100644 --- a/apps/portal-web/src/pageComponents/job/RunningJobTable.tsx +++ b/apps/portal-web/src/pageComponents/job/RunningJobTable.tsx @@ -20,12 +20,13 @@ import { useAsync } from "react-async"; import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { FilterFormContainer } from "src/components/FilterFormContainer"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { runningJobId, RunningJobInfo } from "src/models/job"; import { RunningJobDrawer } from "src/pageComponents/job/RunningJobDrawer"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { Cluster } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; interface FilterForm { jobId: number | undefined; @@ -42,12 +43,16 @@ export const RunningJobQueryTable: React.FC = ({ userId, }) => { - const { defaultCluster } = useStore(DefaultClusterStore); + const { currentClusters, defaultCluster } = useStore(ClusterInfoStore); + + if (!defaultCluster && currentClusters.length === 0) { + return ; + } const [query, setQuery] = useState(() => { return { jobId: undefined, - cluster: defaultCluster, + cluster: defaultCluster ?? currentClusters[0], }; }); diff --git a/apps/portal-web/src/pageComponents/job/SubmitJobForm.tsx b/apps/portal-web/src/pageComponents/job/SubmitJobForm.tsx index 78370ff1ff..b6522d4559 100644 --- a/apps/portal-web/src/pageComponents/job/SubmitJobForm.tsx +++ b/apps/portal-web/src/pageComponents/job/SubmitJobForm.tsx @@ -21,11 +21,12 @@ import { useStore } from "simstate"; import { api } from "src/apis"; import { SingleClusterSelector } from "src/components/ClusterSelector"; import { CodeEditor } from "src/components/CodeEditor"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { AccountSelector } from "src/pageComponents/job/AccountSelector"; import { FileSelectModal } from "src/pageComponents/job/FileSelectModal"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; -import { Cluster, publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; import { formatSize } from "src/utils/format"; interface JobForm { @@ -193,9 +194,16 @@ export const SubmitJobForm: React.FC = ({ initial = initialValues, submit } }, [currentPartitionInfo]); - const { defaultCluster: currentDefaultCluster } = useStore(DefaultClusterStore); + const { currentClusters, defaultCluster } = useStore(ClusterInfoStore); + + // 没有可用集群的情况不再渲染 + if (!defaultCluster && currentClusters.length === 0) { + return ; + } + // 判断是使用template中的cluster还是系统默认cluster，防止系统配置文件更改时仍选改动前的cluster - const defaultCluster = publicConfig.CLUSTERS.find((x) => x.id === initial.cluster?.id) ?? currentDefaultCluster; + const currentQueryCluster = currentClusters.find((x) => x.id === initial.cluster?.id) ?? + (defaultCluster ?? currentClusters[0]); const memorySize = (currentPartitionInfo ? currentPartitionInfo.gpus ? nodeCount * gpuCount @@ -225,7 +233,7 @@ export const SubmitJobForm: React.FC = ({ initial = initialValues, submit form={form} initialValues={{ ...initial, - cluster: defaultCluster, + cluster: currentQueryCluster, }} onFinish={submit} > @@ -377,7 +385,7 @@ export const SubmitJobForm: React.FC = ({ initial = initialValues, submit form.setFields([{ name: "workingDirectory", value: path, touched: true }]); form.validateFields(["workingDirectory"]); }} - cluster={cluster || defaultCluster} + cluster={cluster || currentQueryCluster} /> ) } diff --git a/apps/portal-web/src/pages/_app.tsx b/apps/portal-web/src/pages/_app.tsx index bd91acb3a7..daa79cf50e 100644 --- a/apps/portal-web/src/pages/_app.tsx +++ b/apps/portal-web/src/pages/_app.tsx @@ -14,12 +14,15 @@ import "nprogress/nprogress.css"; import "antd/dist/reset.css"; import { failEvent } from "@ddadaal/next-typed-api-routes-runtime/lib/client"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; import { UiExtensionStore } from "@scow/lib-web/build/extensions/UiExtensionStore"; import { DarkModeCookie, DarkModeProvider, getDarkModeCookieValue } from "@scow/lib-web/build/layouts/darkMode"; import { GlobalStyle } from "@scow/lib-web/build/layouts/globalStyle"; +import { getSortedClusterIds } from "@scow/lib-web/build/utils/cluster"; import { getHostname } from "@scow/lib-web/build/utils/getHostname"; import { useConstant } from "@scow/lib-web/build/utils/hooks"; import { isServer } from "@scow/lib-web/build/utils/isServer"; +import { formatActivatedClusters } from "@scow/lib-web/build/utils/misCommon/clustersActivation"; import { getCurrentLanguageId, getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; import { App as AntdApp } from "antd"; import type { AppContext, AppProps } from "next/app"; @@ -38,12 +41,13 @@ import zh_cn from "src/i18n/zh_cn"; import { AntdConfigProvider } from "src/layouts/AntdConfigProvider"; import { BaseLayout } from "src/layouts/BaseLayout"; import { FloatButtons } from "src/layouts/FloatButtons"; -import { DefaultClusterStore } from "src/stores/DefaultClusterStore"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { LoginNodeStore } from "src/stores/LoginNodeStore"; import { User, UserStore, } from "src/stores/UserStore"; -import { LoginNode, publicConfig, runtimeConfig } from "src/utils/config"; +import { Cluster, getPublicConfigClusters, LoginNode } from "src/utils/cluster"; +import { publicConfig, runtimeConfig } from "src/utils/config"; const languagesMap = { "zh_cn": zh_cn, @@ -53,6 +57,7 @@ const languagesMap = { const FailEventHandler: React.FC = () => { const { message } = AntdApp.useApp(); const userStore = useStore(UserStore); + const { publicConfigClusters, setCurrentClusters } = useStore(ClusterInfoStore); const tArgs = useI18nTranslate(); const languageId = useI18n().currentLanguage.id; @@ -60,6 +65,7 @@ const FailEventHandler: React.FC = () => { // 所以不需要每次userStore变化时来重新注册handler useEffect(() => { failEvent.register((e) => { + if (e.status === 401) { userStore.logout(); return; @@ -87,7 +93,7 @@ const FailEventHandler: React.FC = () => { const clusterId = e.data.clusterErrorsArray[0].clusterId; const clusterName = clusterId ? - (publicConfig.CLUSTERS.find((c) => c.id === clusterId)?.name ?? clusterId) : undefined; + (publicConfigClusters.find((c) => c.id === clusterId)?.name ?? clusterId) : undefined; message.error(`${tArgs("pages._app.adapterConnectionError", [getI18nConfigCurrentText(clusterName, languageId)])}(${ @@ -96,6 +102,27 @@ const FailEventHandler: React.FC = () => { return; } + + if (e.data?.code === "NO_ACTIVATED_CLUSTERS") { + message.error(tArgs("pages._app.noActivatedClusters")); + setCurrentClusters([]); + return; + } + + if (e.data?.code === "NOT_EXIST_IN_ACTIVATED_CLUSTERS") { + message.error(tArgs("pages._app.notExistInActivatedClusters")); + + const currentActivatedClusterIds = e.data.currentActivatedClusterIds; + const activatedClusters = publicConfigClusters.filter((x) => currentActivatedClusterIds.includes(x.id)); + setCurrentClusters(activatedClusters); + return; + } + + if (e.data?.code === "NO_CLUSTERS") { + message.error(tArgs("pages._app.noClusters")); + return; + } + message.error(`${tArgs("pages._app.otherError")}(${e.status}, ${e.data?.code}))`); }); }, []); @@ -118,6 +145,8 @@ interface ExtraProps { loginNodes: Record; darkModeCookieValue: DarkModeCookie | undefined; initialLanguage: string; + clusterConfigs: { [clusterId: string]: ClusterConfigSchema }; + initialCurrentClusters: Cluster[]; } type Props = AppProps & { extra: ExtraProps }; @@ -132,10 +161,13 @@ function MyApp({ Component, pageProps, extra }: Props) { return store; }); + const clusterInfoStore = useConstant(() => { + return createStore(ClusterInfoStore, extra.clusterConfigs, extra.initialCurrentClusters); + }); + const loginNodeStore = useConstant(() => createStore(LoginNodeStore, loginNodes, extra.initialLanguage)); - const defaultClusterStore = useConstant(() => createStore(DefaultClusterStore)); const uiExtensionStore = useConstant(() => createStore(UiExtensionStore, publicConfig.UI_EXTENSION)); // Use the layout defined at the page level, if available @@ -166,7 +198,9 @@ function MyApp({ Component, pageProps, extra }: Props) { definitions: languagesMap[extra.initialLanguage], }} > - + @@ -197,6 +231,8 @@ MyApp.getInitialProps = async (appContext: AppContext) => { darkModeCookieValue: getDarkModeCookieValue(appContext.ctx.req), loginNodes: {}, initialLanguage: "", + clusterConfigs: {}, + initialCurrentClusters: [], }; // This is called on server on first load, and on client on every page transition @@ -219,6 +255,36 @@ MyApp.getInitialProps = async (appContext: AppContext) => { ...userInfo, token: token, }; + + // get cluster configs from config file + const data = await api.getClusterConfigFiles({ query: { token } }) + .then((x) => x, () => ({ clusterConfigs: {} })); + + const clusterConfigs = data?.clusterConfigs; + if (clusterConfigs && Object.keys(clusterConfigs).length > 0) { + + extra.clusterConfigs = clusterConfigs; + const publicConfigClusters + = Object.values(getPublicConfigClusters(clusterConfigs)); + + // get current initial activated clusters + const currentClusters = + await api.getClustersRuntimeInfo({ query: { token } }).then((x) => x, () => undefined); + const initialActivatedClusters = formatActivatedClusters({ + clustersRuntimeInfo: currentClusters?.results, + configClusters: publicConfigClusters as Cluster[], + misDeployed: publicConfig.MIS_DEPLOYED }); + extra.initialCurrentClusters = initialActivatedClusters.activatedClusters ?? []; + + // use all clusters in config files + const clusterSortedIdList = getSortedClusterIds(clusterConfigs); + extra.loginNodes = clusterSortedIdList.reduce((acc, cluster) => { + acc[cluster] = clusterConfigs[cluster].loginNodes; + return acc; + }, {}); + + } + } } @@ -231,19 +297,13 @@ MyApp.getInitialProps = async (appContext: AppContext) => { ?? (hostname && runtimeConfig.UI_CONFIG?.footer?.hostnameTextMap?.[hostname]) ?? runtimeConfig.UI_CONFIG?.footer?.defaultText ?? ""; - extra.loginNodes = publicConfig.CLUSTER_SORTED_ID_LIST.reduce((acc, cluster) => { - acc[cluster] = runtimeConfig.CLUSTERS_CONFIG[cluster].loginNodes; - return acc; - }, {}); - // 从Cookies或header中获取语言id extra.initialLanguage = getCurrentLanguageId(appContext.ctx.req, publicConfig.SYSTEM_LANGUAGE_CONFIG); + } const appProps = await NextApp.getInitialProps(appContext); - // getAvailable - return { ...appProps, extra } as Props; }; diff --git a/apps/portal-web/src/pages/api/app/checkConnectivity.ts b/apps/portal-web/src/pages/api/app/checkConnectivity.ts index 64201edcf9..44f3317eb3 100644 --- a/apps/portal-web/src/pages/api/app/checkConnectivity.ts +++ b/apps/portal-web/src/pages/api/app/checkConnectivity.ts @@ -10,11 +10,12 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; -import { runtimeConfig } from "src/utils/config"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { isPortReachable } from "src/utils/isPortReachable"; +import { route } from "src/utils/route"; export const CheckAppConnectivitySchema = typeboxRouteSchema({ method: "GET", @@ -34,7 +35,7 @@ const auth = authenticate(() => true); const TIMEOUT_MS = 3000; -export default /* #__PURE__*/typeboxRoute(CheckAppConnectivitySchema, async (req, res) => { +export default /* #__PURE__*/route(CheckAppConnectivitySchema, async (req, res) => { const info = await auth(req, res); @@ -42,8 +43,9 @@ export default /* #__PURE__*/typeboxRoute(CheckAppConnectivitySchema, async (req const { host, port, cluster } = req.query; + const clusterConfigs = await getClusterConfigFiles(); // TODO ignore proxy gateway - const proxyGateway = runtimeConfig.CLUSTERS_CONFIG[cluster].proxyGateway; + const proxyGateway = clusterConfigs[cluster].proxyGateway; if (proxyGateway) { return { 200: { ok: true } }; diff --git a/apps/portal-web/src/pages/api/app/connectToApp.ts b/apps/portal-web/src/pages/api/app/connectToApp.ts index 48245aa974..5dc7a5065d 100644 --- a/apps/portal-web/src/pages/api/app/connectToApp.ts +++ b/apps/portal-web/src/pages/api/app/connectToApp.ts @@ -10,13 +10,14 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { status } from "@grpc/grpc-js"; import { AppServiceClient, WebAppProps_ProxyType } from "@scow/protos/build/portal/app"; import { Static, Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; import { handlegRPCError } from "src/utils/server"; // Cannot use ServerConnectPropsConfig from appConfig package @@ -69,7 +70,7 @@ export const ConnectToAppSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(ConnectToAppSchema, async (req, res) => { +export default /* #__PURE__*/route(ConnectToAppSchema, async (req, res) => { const info = await auth(req, res); diff --git a/apps/portal-web/src/pages/api/app/getAppLastSubmission.ts b/apps/portal-web/src/pages/api/app/getAppLastSubmission.ts index 1fe629b606..6d76c2919c 100644 --- a/apps/portal-web/src/pages/api/app/getAppLastSubmission.ts +++ b/apps/portal-web/src/pages/api/app/getAppLastSubmission.ts @@ -10,12 +10,13 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { AppServiceClient } from "@scow/protos/build/portal/app"; import { Static, Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; // Cannot use SubmissionInfo from protos export const SubmissionInfo = Type.Object({ @@ -54,7 +55,7 @@ export const GetAppLastSubmissionSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(GetAppLastSubmissionSchema, async (req, res) => { +export default /* #__PURE__*/route(GetAppLastSubmissionSchema, async (req, res) => { const info = await auth(req, res); diff --git a/apps/portal-web/src/pages/api/app/getAppMetadata.ts b/apps/portal-web/src/pages/api/app/getAppMetadata.ts index c6f37212e7..8d370229c1 100644 --- a/apps/portal-web/src/pages/api/app/getAppMetadata.ts +++ b/apps/portal-web/src/pages/api/app/getAppMetadata.ts @@ -10,15 +10,15 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { status } from "@grpc/grpc-js"; -import { I18nStringType } from "@scow/config/build/i18n"; -import { appCustomAttribute_AttributeTypeToJSON, - AppServiceClient, I18nStringProtoType } from "@scow/protos/build/portal/app"; +import { getI18nTypeFormat } from "@scow/lib-web/src/utils/typeConversion"; +import { appCustomAttribute_AttributeTypeToJSON, AppServiceClient } from "@scow/protos/build/portal/app"; import { Static, Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; import { handlegRPCError } from "src/utils/server"; export const I18nStringSchemaType = Type.Union([ @@ -60,27 +60,6 @@ export const AppCustomAttribute = Type.Object({ }); export type AppCustomAttribute = Static; -// protobuf中定义的grpc返回值的类型映射到前端I18nStringType -const getI18nTypeFormat = (i18nProtoType: I18nStringProtoType | undefined): I18nStringType => { - - if (!i18nProtoType?.value) return ""; - - if (i18nProtoType.value.$case === "directString") { - return i18nProtoType.value.directString; - } else { - const i18nObj = i18nProtoType.value.i18nObject.i18n; - if (!i18nObj) return ""; - return { - i18n: { - default: i18nObj.default, - en: i18nObj.en, - zh_cn: i18nObj.zhCn, - }, - }; - } - -}; - export const GetAppMetadataSchema = typeboxRouteSchema({ method: "GET", @@ -103,7 +82,7 @@ export const GetAppMetadataSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(GetAppMetadataSchema, async (req, res) => { +export default /* #__PURE__*/route(GetAppMetadataSchema, async (req, res) => { const info = await auth(req, res); diff --git a/apps/portal-web/src/pages/api/app/listAvailableApps.ts b/apps/portal-web/src/pages/api/app/listAvailableApps.ts index 790b76f7ee..b78517b5b8 100644 --- a/apps/portal-web/src/pages/api/app/listAvailableApps.ts +++ b/apps/portal-web/src/pages/api/app/listAvailableApps.ts @@ -10,11 +10,12 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { AppServiceClient } from "@scow/protos/build/portal/app"; import { Static, Type } from "@sinclair/typebox"; import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; // Cannot use App from protos export const App = Type.Object({ @@ -52,7 +53,7 @@ export const ListAvailableAppsSchema = typeboxRouteSchema({ // // For now, the API requires token from query // and authenticate manually -export default /* #__PURE__*/typeboxRoute(ListAvailableAppsSchema, async (req) => { +export default /* #__PURE__*/route(ListAvailableAppsSchema, async (req) => { const { cluster } = req.query; diff --git a/apps/portal-web/src/pages/api/desktop/createDesktop.ts b/apps/portal-web/src/pages/api/desktop/createDesktop.ts index f694e950ec..39f86808a9 100644 --- a/apps/portal-web/src/pages/api/desktop/createDesktop.ts +++ b/apps/portal-web/src/pages/api/desktop/createDesktop.ts @@ -10,16 +10,18 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { status } from "@grpc/grpc-js"; import { DesktopServiceClient } from "@scow/protos/build/portal/desktop"; import { Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; import { OperationResult, OperationType } from "src/models/operationLog"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { callLog } from "src/server/operationLog"; import { getClient } from "src/utils/client"; -import { getLoginDesktopEnabled } from "src/utils/config"; +import { getLoginDesktopEnabled } from "src/utils/cluster"; +import { route } from "src/utils/route"; import { handlegRPCError, parseIp } from "src/utils/server"; export const CreateDesktopSchema = typeboxRouteSchema({ @@ -58,11 +60,12 @@ export const CreateDesktopSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(CreateDesktopSchema, async (req, res) => { +export default /* #__PURE__*/route(CreateDesktopSchema, async (req, res) => { const { cluster, loginNode, wm, desktopName } = req.body; - const loginDesktopEnabled = getLoginDesktopEnabled(cluster); + const clusterConfigs = await getClusterConfigFiles(); + const loginDesktopEnabled = getLoginDesktopEnabled(cluster, clusterConfigs); if (!loginDesktopEnabled) { return { 501: { code: "CLUSTER_LOGIN_DESKTOP_NOT_ENABLED" as const } }; diff --git a/apps/portal-web/src/pages/api/desktop/killDesktop.ts b/apps/portal-web/src/pages/api/desktop/killDesktop.ts index dde084a803..c40f4a9069 100644 --- a/apps/portal-web/src/pages/api/desktop/killDesktop.ts +++ b/apps/portal-web/src/pages/api/desktop/killDesktop.ts @@ -10,15 +10,17 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { DesktopServiceClient } from "@scow/protos/build/portal/desktop"; import { Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; import { OperationResult, OperationType } from "src/models/operationLog"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { callLog } from "src/server/operationLog"; import { getClient } from "src/utils/client"; -import { getLoginDesktopEnabled } from "src/utils/config"; +import { getLoginDesktopEnabled } from "src/utils/cluster"; +import { route } from "src/utils/route"; import { parseIp } from "src/utils/server"; export const KillDesktopSchema = typeboxRouteSchema({ @@ -40,11 +42,12 @@ export const KillDesktopSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(KillDesktopSchema, async (req, res) => { +export default /* #__PURE__*/route(KillDesktopSchema, async (req, res) => { const { cluster, loginNode, displayId } = req.body; - const loginDesktopEnabled = getLoginDesktopEnabled(cluster); + const clusterConfigs = await getClusterConfigFiles(); + const loginDesktopEnabled = getLoginDesktopEnabled(cluster, clusterConfigs); if (!loginDesktopEnabled) { return { 501: { code: "CLUSTER_LOGIN_DESKTOP_NOT_ENABLED" as const } }; diff --git a/apps/portal-web/src/pages/api/desktop/launchDesktop.ts b/apps/portal-web/src/pages/api/desktop/launchDesktop.ts index 1a7cbad49e..109a1d1e8b 100644 --- a/apps/portal-web/src/pages/api/desktop/launchDesktop.ts +++ b/apps/portal-web/src/pages/api/desktop/launchDesktop.ts @@ -10,13 +10,15 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { DesktopServiceClient } from "@scow/protos/build/portal/desktop"; import { Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { getClient } from "src/utils/client"; -import { getLoginDesktopEnabled } from "src/utils/config"; +import { getLoginDesktopEnabled } from "src/utils/cluster"; +import { route } from "src/utils/route"; import { handlegRPCError } from "src/utils/server"; export const LaunchDesktopSchema = typeboxRouteSchema({ @@ -43,10 +45,11 @@ export const LaunchDesktopSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(LaunchDesktopSchema, async (req, res) => { +export default /* #__PURE__*/route(LaunchDesktopSchema, async (req, res) => { const { cluster, loginNode, displayId } = req.body; - const loginDesktopEnabled = getLoginDesktopEnabled(cluster); + const clusterConfigs = await getClusterConfigFiles(); + const loginDesktopEnabled = getLoginDesktopEnabled(cluster, clusterConfigs); if (!loginDesktopEnabled) { return { 501: { code: "CLUSTER_LOGIN_DESKTOP_NOT_ENABLED" as const } }; diff --git a/apps/portal-web/src/pages/api/desktop/listAvailableWms.ts b/apps/portal-web/src/pages/api/desktop/listAvailableWms.ts index 3d91b7457c..d305d1a856 100644 --- a/apps/portal-web/src/pages/api/desktop/listAvailableWms.ts +++ b/apps/portal-web/src/pages/api/desktop/listAvailableWms.ts @@ -10,13 +10,15 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { DesktopServiceClient } from "@scow/protos/build/portal/desktop"; import { Static, Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { getClient } from "src/utils/client"; -import { getLoginDesktopEnabled } from "src/utils/config"; +import { getLoginDesktopEnabled } from "src/utils/cluster"; +import { route } from "src/utils/route"; // Cannot use AvailableWm from protos export const AvailableWm = Type.Object({ @@ -44,12 +46,13 @@ export const ListAvailableWmsSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(ListAvailableWmsSchema, async (req, res) => { +export default /* #__PURE__*/route(ListAvailableWmsSchema, async (req, res) => { const { cluster } = req.query; - const loginDesktopEnabled = getLoginDesktopEnabled(cluster); + const clusterConfigs = await getClusterConfigFiles(); + const loginDesktopEnabled = getLoginDesktopEnabled(cluster, clusterConfigs); if (!loginDesktopEnabled) { return { 501: { code: "CLUSTER_LOGIN_DESKTOP_NOT_ENABLED" as const } }; diff --git a/apps/portal-web/src/pages/api/desktop/listDesktops.ts b/apps/portal-web/src/pages/api/desktop/listDesktops.ts index fe0c7c0a31..41f01deda6 100644 --- a/apps/portal-web/src/pages/api/desktop/listDesktops.ts +++ b/apps/portal-web/src/pages/api/desktop/listDesktops.ts @@ -10,13 +10,15 @@ * See the Mulan PSL v2 for more details. */ -import { typeboxRoute, typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; import { DesktopServiceClient } from "@scow/protos/build/portal/desktop"; import { Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { getClient } from "src/utils/client"; -import { getLoginDesktopEnabled } from "src/utils/config"; +import { getLoginDesktopEnabled } from "src/utils/cluster"; +import { route } from "src/utils/route"; export const ListDesktopsSchema = typeboxRouteSchema({ method: "GET", @@ -46,11 +48,13 @@ export const ListDesktopsSchema = typeboxRouteSchema({ const auth = authenticate(() => true); -export default /* #__PURE__*/typeboxRoute(ListDesktopsSchema, async (req, res) => { +export default /* #__PURE__*/route(ListDesktopsSchema, async (req, res) => { const { cluster, loginNode } = req.query; - const loginDesktopEnabled = getLoginDesktopEnabled(cluster); + const clusterConfigs = await getClusterConfigFiles(); + + const loginDesktopEnabled = getLoginDesktopEnabled(cluster, clusterConfigs); if (!loginDesktopEnabled) { return { 501: { code: "CLUSTER_LOGIN_DESKTOP_NOT_ENABLED" as const } }; } diff --git a/apps/portal-web/src/pages/api/file/listAvailableTransferClusters.ts b/apps/portal-web/src/pages/api/file/listAvailableTransferClusters.ts index f41eb2be47..3a33d615e2 100644 --- a/apps/portal-web/src/pages/api/file/listAvailableTransferClusters.ts +++ b/apps/portal-web/src/pages/api/file/listAvailableTransferClusters.ts @@ -11,9 +11,12 @@ */ import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { libGetClustersRuntimeInfo } from "@scow/lib-web/build/server/clustersActivation"; import { getCurrentLanguageId, getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; +import { ClusterActivationStatus } from "@scow/protos/build/server/config"; import { Static, Type } from "@sinclair/typebox"; import { authenticate } from "src/auth/server"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { publicConfig, runtimeConfig } from "src/utils/config"; import { route } from "src/utils/route"; @@ -24,7 +27,6 @@ export const Cluster = Type.Object({ export type ClusterInfo = Static; - export const ListAvailableTransferClustersSchema = typeboxRouteSchema({ method: "GET", @@ -49,11 +51,17 @@ export default route(ListAvailableTransferClustersSchema, async (req, res) => { if (!info) { return; } - const clusterList: ClusterInfo[] = publicConfig.CLUSTERS - .filter((x) => runtimeConfig.CLUSTERS_CONFIG[x.id].crossClusterFileTransfer?.enabled) + const clusterConfigs = await getClusterConfigFiles(); + + const clustersRuntimeInfo = await libGetClustersRuntimeInfo( + publicConfig.MIS_SERVER_URL, runtimeConfig.SCOW_API_AUTH_TOKEN); + + const clusterList: ClusterInfo[] = clustersRuntimeInfo + .filter((x) => x.activationStatus === ClusterActivationStatus.ACTIVATED + && clusterConfigs[x.clusterId]?.crossClusterFileTransfer?.enabled) .map((x) => ({ - id: x.id, - name: getI18nConfigCurrentText(x.name, languageId), + id: x.clusterId, + name: getI18nConfigCurrentText(clusterConfigs[x.clusterId].displayName, languageId), })); diff --git a/apps/portal-web/src/pages/api/getClusterConfigFiles.ts b/apps/portal-web/src/pages/api/getClusterConfigFiles.ts new file mode 100644 index 0000000000..6ee048ab70 --- /dev/null +++ b/apps/portal-web/src/pages/api/getClusterConfigFiles.ts @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { ClusterConfigSchema } from "@scow/config/build/cluster"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { validateToken } from "src/auth/token"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; +import { route } from "src/utils/route"; + + + +export const getClusterConfigFilesSchema = typeboxRouteSchema({ + method: "GET", + + // only set the query value when firstly used in getInitialProps + query: Type.Object({ + token: Type.Optional(Type.String()), + }), + + responses: { + + 200: Type.Object({ + clusterConfigs: Type.Record(Type.String(), ClusterConfigSchema) }), + + 403: Type.Null(), + }, +}); + + +const auth = authenticate(() => true); + +export default route(getClusterConfigFilesSchema, + async (req, res) => { + + const { token } = req.query; + // when firstly used in getInitialProps, check the token + // when logged in, use auth() + const info = token ? await validateToken(token) : await auth(req, res); + + if (!info) { return { 403: null }; } + + const modifiedClusters: Record = await getClusterConfigFiles(); + + return { + 200: { clusterConfigs: modifiedClusters }, + }; + }); diff --git a/apps/portal-web/src/pages/api/getClustersRuntimeInfo.ts b/apps/portal-web/src/pages/api/getClustersRuntimeInfo.ts new file mode 100644 index 0000000000..72da263b04 --- /dev/null +++ b/apps/portal-web/src/pages/api/getClustersRuntimeInfo.ts @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { ClusterRuntimeInfoSchema } from "@scow/config/build/type"; +import { libGetClustersRuntimeInfo } from "@scow/lib-web/build/server/clustersActivation"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { validateToken } from "src/auth/token"; +import { publicConfig, runtimeConfig } from "src/utils/config"; +import { route } from "src/utils/route"; + +export const GetClustersRuntimeInfoSchema = typeboxRouteSchema({ + + method: "GET", + + // only set the query value when firstly used in getInitialProps + query: Type.Object({ + token: Type.Optional(Type.String()), + }), + + responses: { + 200: Type.Object({ + results: Type.Array(ClusterRuntimeInfoSchema), + }), + + 403: Type.Null(), + }, +}); + +const auth = authenticate(() => true); +export default route(GetClustersRuntimeInfoSchema, + async (req, res) => { + const { token } = req.query; + // when firstly used in getInitialProps, check the token + // when logged in, use auth() + const info = token ? await validateToken(token) : await auth(req, res); + if (!info) { return { 403: null }; } + + const reply = await libGetClustersRuntimeInfo(publicConfig.MIS_SERVER_URL, runtimeConfig.SCOW_API_AUTH_TOKEN); + return { + 200: { + results: reply, + }, + }; + }); diff --git a/apps/portal-web/src/pages/api/proxy/[clusterId]/[type]/[node]/[port]/[[...path]].ts b/apps/portal-web/src/pages/api/proxy/[clusterId]/[type]/[node]/[port]/[[...path]].ts index 440a960285..d295ae9f7f 100644 --- a/apps/portal-web/src/pages/api/proxy/[clusterId]/[type]/[node]/[port]/[[...path]].ts +++ b/apps/portal-web/src/pages/api/proxy/[clusterId]/[type]/[node]/[port]/[[...path]].ts @@ -12,6 +12,7 @@ import { NextApiRequest, NextApiResponse } from "next"; import { checkCookie } from "src/auth/server"; +import { getClusterConfigFiles } from "src/server/clusterConfig"; import { parseProxyTarget, proxy } from "src/server/setup/proxy"; export default async (req: NextApiRequest, res: NextApiResponse) => { @@ -26,8 +27,10 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { return; } + const clusterConfigs = await getClusterConfigFiles(); + // req.url of next.js removes base path - const target = parseProxyTarget(req.url!, false); + const target = parseProxyTarget(req.url!, false, clusterConfigs); if (target instanceof Error) { res.status(400).send(target.message); diff --git a/apps/portal-web/src/pages/apps/[clusterId]/create/[app].tsx b/apps/portal-web/src/pages/apps/[clusterId]/create/[app].tsx index d485e5be81..53b28546e4 100644 --- a/apps/portal-web/src/pages/apps/[clusterId]/create/[app].tsx +++ b/apps/portal-web/src/pages/apps/[clusterId]/create/[app].tsx @@ -17,11 +17,14 @@ import { NextPage } from "next"; import { useRouter } from "next/router"; import { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { PageTitle } from "src/components/PageTitle"; import { useI18nTranslateToString } from "src/i18n"; import { LaunchAppForm } from "src/pageComponents/app/LaunchAppForm"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; @@ -34,6 +37,11 @@ export const AppIndexPage: NextPage = requireAuth(() => true)(() => { const { message } = App.useApp(); const t = useI18nTranslateToString(); + const { currentClusters } = useStore(ClusterInfoStore); + if (!currentClusters.find((x) => x.id === clusterId)) { + return ; + } + const { data, isLoading } = useAsync({ promiseFn: useCallback(async () => { return await api.getAppMetadata({ query: { appId, cluster: clusterId } }) diff --git a/apps/portal-web/src/pages/apps/[clusterId]/createApps.tsx b/apps/portal-web/src/pages/apps/[clusterId]/createApps.tsx index 36bc9acee1..d28047c610 100644 --- a/apps/portal-web/src/pages/apps/[clusterId]/createApps.tsx +++ b/apps/portal-web/src/pages/apps/[clusterId]/createApps.tsx @@ -15,11 +15,12 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLangua import { Result } from "antd"; import { NextPage } from "next"; import { useRouter } from "next/router"; +import { useStore } from "simstate"; import { requireAuth } from "src/auth/requireAuth"; import { PageTitle } from "src/components/PageTitle"; import { useI18n, useI18nTranslate, useI18nTranslateToString } from "src/i18n"; import { CreateAppsTable } from "src/pageComponents/app/CreateAppsTable"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; export const CreateAppsIndexPage: NextPage = requireAuth(() => true)(() => { @@ -28,7 +29,9 @@ export const CreateAppsIndexPage: NextPage = requireAuth(() => true)(() => { const router = useRouter(); const clusterId = queryToString(router.query.clusterId); - const cluster = publicConfig.CLUSTERS.find((x) => x.id === clusterId); + + const { currentClusters } = useStore(ClusterInfoStore); + const cluster = currentClusters.find((x) => x.id === clusterId); const tArgs = useI18nTranslate(); const t = useI18nTranslateToString(); diff --git a/apps/portal-web/src/pages/apps/[clusterId]/sessions.tsx b/apps/portal-web/src/pages/apps/[clusterId]/sessions.tsx index 2ba79e8e5e..62d17d7df0 100644 --- a/apps/portal-web/src/pages/apps/[clusterId]/sessions.tsx +++ b/apps/portal-web/src/pages/apps/[clusterId]/sessions.tsx @@ -15,11 +15,12 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLangua import { Result } from "antd"; import { NextPage } from "next"; import { useRouter } from "next/router"; +import { useStore } from "simstate"; import { requireAuth } from "src/auth/requireAuth"; import { PageTitle } from "src/components/PageTitle"; import { useI18n, useI18nTranslate, useI18nTranslateToString } from "src/i18n"; import { AppSessionsTable } from "src/pageComponents/app/AppSessionsTable"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; export const SessionsIndexPage: NextPage = requireAuth(() => true)(() => { @@ -27,7 +28,9 @@ export const SessionsIndexPage: NextPage = requireAuth(() => true)(() => { const languageId = useI18n().currentLanguage.id; const router = useRouter(); const clusterId = queryToString(router.query.clusterId); - const cluster = publicConfig.CLUSTERS.find((x) => x.id === clusterId); + + const { currentClusters } = useStore(ClusterInfoStore); + const cluster = currentClusters.find((x) => x.id === clusterId); const tArgs = useI18nTranslate(); const t = useI18nTranslateToString(); diff --git a/apps/portal-web/src/pages/dashboard.tsx b/apps/portal-web/src/pages/dashboard.tsx index 51b9393ab4..8c20c8d592 100644 --- a/apps/portal-web/src/pages/dashboard.tsx +++ b/apps/portal-web/src/pages/dashboard.tsx @@ -21,8 +21,8 @@ import { requireAuth } from "src/auth/requireAuth"; import { useI18nTranslateToString } from "src/i18n"; import { OverviewTable } from "src/pageComponents/dashboard/OverviewTable"; import { QuickEntry } from "src/pageComponents/dashboard/QuickEntry"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { UserStore } from "src/stores/UserStore"; -import { publicConfig } from "src/utils/config"; import { Head } from "src/utils/head"; import { styled } from "styled-components"; @@ -44,12 +44,12 @@ export const DashboardPage: NextPage = requireAuth(() => true)(() => { const t = useI18nTranslateToString(); + const { publicConfigClusters, currentClusters } = useStore(ClusterInfoStore); + const { data, isLoading } = useAsync({ promiseFn: useCallback(async () => { - const clusters = publicConfig.CLUSTERS; - - const rawClusterInfoPromises = clusters.map((x) => + const rawClusterInfoPromises = currentClusters.map((x) => api.getClusterRunningInfo({ query: { clusterId: x.id } }) .httpError(500, () => {}), ); @@ -66,7 +66,7 @@ export const DashboardPage: NextPage = requireAuth(() => true)(() => { return { ...result, value:{ - clusterInfo:{ clusterName:clusters[idx].id, + clusterInfo:{ clusterName: currentClusters[idx].id, partitions:result.value.clusterInfo.partitions }, }, } as PromiseSettledResult; @@ -81,13 +81,13 @@ export const DashboardPage: NextPage = requireAuth(() => true)(() => { // 处理失败的结果 - const failedClusters = clusters.filter((x) => + const failedClusters = currentClusters.filter((x) => !successfulResults.find((y) => y.clusterInfo.clusterName === x.id), ); const clustersInfo = successfulResults .map((cluster) => ({ clusterInfo: { ...cluster.clusterInfo, - clusterName: clusters.find((x) => x.id === cluster.clusterInfo.clusterName)?.name } })) + clusterName: currentClusters.find((x) => x.id === cluster.clusterInfo.clusterName)?.name } })) .flatMap((cluster) => cluster.clusterInfo.partitions.map((x) => ({ clusterName: cluster.clusterInfo.clusterName, @@ -109,7 +109,7 @@ export const DashboardPage: NextPage = requireAuth(() => true)(() => { return ( - + ({ ...item, id:idx })) : []} diff --git a/apps/portal-web/src/pages/desktop/index.tsx b/apps/portal-web/src/pages/desktop/index.tsx index e7d531fa5d..4b068d1753 100644 --- a/apps/portal-web/src/pages/desktop/index.tsx +++ b/apps/portal-web/src/pages/desktop/index.tsx @@ -10,14 +10,21 @@ * See the Mulan PSL v2 for more details. */ +import { getSortedClusterIds } from "@scow/config/build/cluster"; +import { ClusterActivationStatus } from "@scow/config/build/type"; import { getCurrentLanguageId, getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; import { GetServerSideProps, NextPage } from "next"; +import { useStore } from "simstate"; +import { api } from "src/apis"; +import { getTokenFromCookie } from "src/auth/cookie"; import { requireAuth } from "src/auth/requireAuth"; import { NotFoundPage } from "src/components/errorPages/NotFoundPage"; import { PageTitle } from "src/components/PageTitle"; import { useI18nTranslateToString } from "src/i18n"; import { DesktopTable } from "src/pageComponents/desktop/DesktopTable"; -import { Cluster, getLoginDesktopEnabled, publicConfig, runtimeConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster, getLoginDesktopEnabled } from "src/utils/cluster"; +import { publicConfig } from "src/utils/config"; import { Head } from "src/utils/head"; type Props = { loginDesktopEnabledClusters: Cluster[]; @@ -26,7 +33,8 @@ type Props = { export const DesktopIndexPage: NextPage = requireAuth(() => true) ((props: Props) => { - if (!publicConfig.ENABLE_LOGIN_DESKTOP) { + const { enableLoginDesktop } = useStore(ClusterInfoStore); + if (!enableLoginDesktop || props.loginDesktopEnabledClusters.length === 0) { return ; } @@ -46,11 +54,24 @@ export const getServerSideProps: GetServerSideProps = async ({ req }) => const languageId = getCurrentLanguageId(req, publicConfig.SYSTEM_LANGUAGE_CONFIG); - const loginDesktopEnabledClusters = publicConfig.CLUSTER_SORTED_ID_LIST - .filter((clusterId) => getLoginDesktopEnabled(clusterId)) + const token = getTokenFromCookie({ req }); + const resp = await api.getClusterConfigFiles({ query: { token } }); + const clusterConfigs = resp.clusterConfigs; + const clusterSortedIdList = getSortedClusterIds(resp.clusterConfigs); + const currentClusters = await api.getClustersRuntimeInfo({ query: { token } }); + + const activatedClusterIds = currentClusters?.results + .filter((x) => x.activationStatus === ClusterActivationStatus.ACTIVATED).map((x) => x.clusterId) ?? []; + const sortedCurrentClusterIds = clusterSortedIdList.filter((id) => activatedClusterIds.includes(id)); + + const sortedClusterIdList = publicConfig.MIS_DEPLOYED ? + sortedCurrentClusterIds : clusterSortedIdList; + + const loginDesktopEnabledClusters = sortedClusterIdList + .filter((clusterId) => getLoginDesktopEnabled(clusterId, clusterConfigs)) .map((clusterId) => ({ id: clusterId, - name: getI18nConfigCurrentText(runtimeConfig.CLUSTERS_CONFIG[clusterId].displayName, languageId) } as Cluster)); + name: getI18nConfigCurrentText(clusterConfigs[clusterId].displayName, languageId) } as Cluster)); return { props: { diff --git a/apps/portal-web/src/pages/files/[cluster]/[[...path]].tsx b/apps/portal-web/src/pages/files/[cluster]/[[...path]].tsx index d46386fa16..65748f4dec 100644 --- a/apps/portal-web/src/pages/files/[cluster]/[[...path]].tsx +++ b/apps/portal-web/src/pages/files/[cluster]/[[...path]].tsx @@ -15,10 +15,11 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLangua import { Result } from "antd"; import { NextPage } from "next"; import { useRouter } from "next/router"; +import { useStore } from "simstate"; import { requireAuth } from "src/auth/requireAuth"; import { useI18n, useI18nTranslateToString } from "src/i18n"; import { FileManager } from "src/pageComponents/filemanager/FileManager"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; export const FileManagerPage: NextPage = requireAuth(() => true)(() => { @@ -32,7 +33,9 @@ export const FileManagerPage: NextPage = requireAuth(() => true)(() => { const t = useI18nTranslateToString(); - const clusterObj = publicConfig.CLUSTERS.find((x) => x.id === cluster); + const { currentClusters } = useStore(ClusterInfoStore); + + const clusterObj = currentClusters.find((x) => x.id === cluster); const fullPath = "/" + pathParts?.join("/") ?? ""; diff --git a/apps/portal-web/src/pages/files/[cluster]/~.tsx b/apps/portal-web/src/pages/files/[cluster]/~.tsx index a91bf132b6..9303b54bec 100644 --- a/apps/portal-web/src/pages/files/[cluster]/~.tsx +++ b/apps/portal-web/src/pages/files/[cluster]/~.tsx @@ -17,10 +17,13 @@ import { useRouter } from "next/router"; import { join } from "path"; import { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { ServerErrorPage } from "src/components/errorPages/ServerErrorPage"; import { Redirect } from "src/components/Redirect"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; export const HomeDirFileManagerPage: NextPage = requireAuth(() => true)(() => { @@ -28,6 +31,11 @@ export const HomeDirFileManagerPage: NextPage = requireAuth(() => true)(() => { const cluster = queryToString(router.query.cluster); + const { currentClusters } = useStore(ClusterInfoStore); + if (!currentClusters.find((x) => x.id === cluster)) { + return ; + } + const { data, isLoading, error } = useAsync({ promiseFn: useCallback(async () => api.getHomeDirectory({ query: { cluster } }), [cluster]), }); diff --git a/apps/portal-web/src/pages/files/currentTransferInfo.tsx b/apps/portal-web/src/pages/files/currentTransferInfo.tsx index a7788c4567..43d5cd1a9f 100644 --- a/apps/portal-web/src/pages/files/currentTransferInfo.tsx +++ b/apps/portal-web/src/pages/files/currentTransferInfo.tsx @@ -11,12 +11,13 @@ */ import { NextPage } from "next"; +import { useStore } from "simstate"; import { requireAuth } from "src/auth/requireAuth"; import { PageTitle } from "src/components/PageTitle"; import { Redirect } from "src/components/Redirect"; import { prefix, useI18nTranslateToString } from "src/i18n"; import { TransferInfoTable } from "src/pageComponents/filemanager/TransferInfoTable"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; const p = prefix("pages.files.currentTransferInfo."); @@ -24,7 +25,9 @@ export const FileTransferPage: NextPage = requireAuth(() => true)(() => { const t = useI18nTranslateToString(); - if (!publicConfig.CROSS_CLUSTER_FILE_TRANSFER_ENABLED) { + const { crossClusterFileTransferEnabled } = useStore(ClusterInfoStore); + + if (!crossClusterFileTransferEnabled) { return ; } diff --git a/apps/portal-web/src/pages/files/fileTransfer.tsx b/apps/portal-web/src/pages/files/fileTransfer.tsx index ed9f5272da..1e09be19ee 100644 --- a/apps/portal-web/src/pages/files/fileTransfer.tsx +++ b/apps/portal-web/src/pages/files/fileTransfer.tsx @@ -15,13 +15,15 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLangua import { App, Button, Col, Row } from "antd"; import { NextPage } from "next"; import { useState } from "react"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; import { PageTitle } from "src/components/PageTitle"; import { Redirect } from "src/components/Redirect"; import { prefix, useI18n, useI18nTranslateToString } from "src/i18n"; import { ClusterFileTable } from "src/pageComponents/filemanager/ClusterFileTable"; -import { Cluster, publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; +import { Cluster } from "src/utils/cluster"; type FileInfoKey = React.Key; @@ -85,6 +87,12 @@ export const FileTransferPage: NextPage = requireAuth(() => true)(() => { const t = useI18nTranslateToString(); + const { crossClusterFileTransferEnabled } = useStore(ClusterInfoStore); + + if (!crossClusterFileTransferEnabled) { + return ; + } + const [clusterLeft, setClusterLeft] = useState(); const [clusterRight, setClusterRight] = useState(); @@ -94,10 +102,6 @@ export const FileTransferPage: NextPage = requireAuth(() => true)(() => { const [selectedKeysLeft, setSelectedKeysLeft] = useState([]); const [selectedKeysRight, setSelectedKeysRight] = useState([]); - if (!publicConfig.CROSS_CLUSTER_FILE_TRANSFER_ENABLED) { - return ; - } - return ( <> diff --git a/apps/portal-web/src/pages/jobs/submit.tsx b/apps/portal-web/src/pages/jobs/submit.tsx index c379d5ab04..bdcd6c18d8 100644 --- a/apps/portal-web/src/pages/jobs/submit.tsx +++ b/apps/portal-web/src/pages/jobs/submit.tsx @@ -16,11 +16,13 @@ import { Spin } from "antd"; import { GetServerSideProps, NextPage } from "next"; import { useCallback } from "react"; import { useAsync } from "react-async"; +import { useStore } from "simstate"; import { api } from "src/apis"; import { requireAuth } from "src/auth/requireAuth"; import { PageTitle } from "src/components/PageTitle"; import { useI18nTranslateToString } from "src/i18n"; import { SubmitJobForm } from "src/pageComponents/job/SubmitJobForm"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { getServerI18nConfigText, publicConfig } from "src/utils/config"; import { Head } from "src/utils/head"; @@ -39,7 +41,10 @@ export const SubmitJobPage: NextPage = requireAuth(() => true)( const { data, isLoading } = useAsync({ promiseFn: useCallback(async () => { if (cluster && jobTemplateId) { - const clusterObj = publicConfig.CLUSTERS.find((x) => x.id === cluster); + + const { currentClusters } = useStore(ClusterInfoStore); + + const clusterObj = currentClusters.find((x) => x.id === cluster); if (!clusterObj) { return undefined; } return api.getJobTemplate({ query: { cluster, id: jobTemplateId } }) .then(({ template }) => ({ diff --git a/apps/portal-web/src/pages/shell/[cluster]/[loginNode]/[[...path]].tsx b/apps/portal-web/src/pages/shell/[cluster]/[loginNode]/[[...path]].tsx index 221fe2193b..5a089a78d5 100644 --- a/apps/portal-web/src/pages/shell/[cluster]/[loginNode]/[[...path]].tsx +++ b/apps/portal-web/src/pages/shell/[cluster]/[loginNode]/[[...path]].tsx @@ -20,8 +20,10 @@ import Router, { useRouter } from "next/router"; import { useRef } from "react"; import { useStore } from "simstate"; import { requireAuth } from "src/auth/requireAuth"; +import { ClusterNotAvailablePage } from "src/components/errorPages/ClusterNotAvailablePage"; import { NotFoundPage } from "src/components/errorPages/NotFoundPage"; import { Localized, useI18n, useI18nTranslateToString } from "src/i18n"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { LoginNodeStore } from "src/stores/LoginNodeStore"; import { publicConfig } from "src/utils/config"; import { Head } from "src/utils/head"; @@ -85,13 +87,19 @@ export const ShellPage: NextPage = requireAuth(() => true)(({ userStore }) => { const loginNode = router.query.loginNode as string; const paths = router.query.path as (string[] | undefined); + const { currentClusters } = useStore(ClusterInfoStore); + + if (!currentClusters.find((x) => x.id === cluster)) { + return ; + } + const { loginNodes } = useStore(LoginNodeStore); const currentLoginNodeName = loginNodes[cluster].find((x) => x.address === loginNode)?.name ?? loginNode; const headerRef = useRef(null); const clusterName = - getI18nConfigCurrentText(publicConfig.CLUSTERS.find((x) => x.id === cluster)?.name || cluster, languageId); + getI18nConfigCurrentText(currentClusters.find((x) => x.id === cluster)?.name || cluster, languageId); const t = useI18nTranslateToString(); diff --git a/apps/portal-web/src/pages/shell/index.tsx b/apps/portal-web/src/pages/shell/index.tsx index 6b5d3463db..cdc2d4fdd3 100644 --- a/apps/portal-web/src/pages/shell/index.tsx +++ b/apps/portal-web/src/pages/shell/index.tsx @@ -13,9 +13,10 @@ import { getI18nConfigCurrentText } from "@scow/lib-web/build/utils/systemLanguage"; import { Button } from "antd"; import { NextPage } from "next"; +import { useStore } from "simstate"; import { requireAuth } from "src/auth/requireAuth"; import { Localized, useI18n, useI18nTranslateToString } from "src/i18n"; -import { publicConfig } from "src/utils/config"; +import { ClusterInfoStore } from "src/stores/ClusterInfoStore"; import { Head } from "src/utils/head"; export const ShellIndexPage: NextPage = requireAuth(() => true)(() => { @@ -23,13 +24,15 @@ export const ShellIndexPage: NextPage = requireAuth(() => true)(() => { const languageId = useI18n().currentLanguage.id; const t = useI18nTranslateToString(); + const { currentClusters } = useStore(ClusterInfoStore); + return (

- {publicConfig.CLUSTERS.map(({ id, name }) => ( + {currentClusters.map(({ id, name }) => (