create a user secret for every job to save user extension

src/database-controller/sdk/index.js

@@ -40,11 +40,12 @@ class DatabaseModel {
           type: Sequelize.DATE,
           allowNull: false,
         },
-        // `dockerSecretDef`, `configSecretDef`, `tokenSecretDef` and `priorityClassDef` is the definition of job add-ons.
+        // `dockerSecretDef`, `configSecretDef`, `userSecretDef`, `tokenSecretDef` and `priorityClassDef` is the definition of job add-ons.
         // They are generated by rest-server and recorded into database by write-merger.
         // These add-ons are created by poller or the short-cut in write-merger.
         dockerSecretDef: Sequelize.TEXT,
         configSecretDef: Sequelize.TEXT,
+        userSecretDef: Sequelize.TEXT,
         tokenSecretDef: Sequelize.TEXT,
         priorityClassDef: Sequelize.TEXT,
         retries: Sequelize.INTEGER,

src/database-controller/src/common/framework.js

@@ -350,10 +350,11 @@ class Snapshot {
 }
 
 // Class Add-ons handles creation/patching/deletion of job add-ons.
-// Currently there are 4 types of add-ons: configSecret, priorityClass, dockerSecret, and tokenSecret.
+// Currently there are 5 types of add-ons: configSecret, userSecret, priorityClass, dockerSecret, and tokenSecret.
 class AddOns {
   constructor(
     configSecretDef = null,
+    userSecretDef = null,
     priorityClassDef = null,
     dockerSecretDef = null,
     tokenSecretDef = null,
@@ -363,6 +364,11 @@ class AddOns {
     } else {
       this._configSecretDef = configSecretDef;
     }
+    if (userSecretDef !== null && !(userSecretDef instanceof Object)) {
+      this._userSecretDef = JSON.parse(userSecretDef);
+    } else {
+      this._userSecretDef = userSecretDef;
+    }
     if (priorityClassDef !== null && !(priorityClassDef instanceof Object)) {
       this._priorityClassDef = JSON.parse(priorityClassDef);
     } else {
@@ -394,6 +400,19 @@ class AddOns {
         }
       }
     }
+    if (this._userSecretDef) {
+      try {
+        await k8s.createSecret(this._userSecretDef);
+      } catch (err) {
+        if (err.response && err.response.statusCode === 409) {
+          logger.warn(
+            `Secret ${this._userSecretDef.metadata.name} already exists.`,
+          );
+        } else {
+          throw err;
+        }
+      }
+    }
     if (this._priorityClassDef) {
       try {
         await k8s.createPriorityClass(this._priorityClassDef);
@@ -441,6 +460,10 @@ class AddOns {
       k8s
         .patchSecretOwnerToFramework(this._configSecretDef, frameworkResponse)
         .catch(logError);
+    this._userSecretDef &&
+      k8s
+        .patchSecretOwnerToFramework(this._userSecretDef, frameworkResponse)
+        .catch(logError);
     this._dockerSecretDef &&
       k8s
         .patchSecretOwnerToFramework(this._dockerSecretDef, frameworkResponse)
@@ -455,6 +478,8 @@ class AddOns {
     // do not await for delete
     this._configSecretDef &&
       k8s.deleteSecret(this._configSecretDef.metadata.name).catch(logError);
+    this._userSecretDef &&
+      k8s.deleteSecret(this._userSecretDef.metadata.name).catch(logError);
     this._priorityClassDef &&
       k8s
         .deletePriorityClass(this._priorityClassDef.metadata.name)
@@ -470,6 +495,9 @@ class AddOns {
     if (this._configSecretDef) {
       update.configSecretDef = JSON.stringify(this._configSecretDef);
     }
+    if (this._userSecretDef) {
+      update.userSecretDef = JSON.stringify(this._userSecretDef);
+    }
     if (this._priorityClassDef) {
       update.priorityClassDef = JSON.stringify(this._priorityClassDef);
     }

src/database-controller/src/poller/index.js

@@ -145,6 +145,7 @@ async function poll() {
       attributes: [
         'name',
         'configSecretDef',
+        'userSecretDef',
         'priorityClassDef',
         'dockerSecretDef',
         'tokenSecretDef',
@@ -165,6 +166,7 @@ async function poll() {
       const snapshot = new Snapshot(framework.snapshot);
       const addOns = new AddOns(
         framework.configSecretDef,
+        framework.userSecretDef,
         framework.priorityClassDef,
         framework.dockerSecretDef,
         framework.tokenSecretDef,

src/database-controller/src/write-merger/handler.js

@@ -95,6 +95,7 @@ async function postWatchEvents(req, res, next) {
             internalUpdate.apiServerDeleted = true;
             // If the job is completed and deleted, we could reset its sensitive fields.
             internalUpdate.configSecretDef = '';
+            internalUpdate.userSecretDef = '';
             internalUpdate.dockerSecretDef = '';
             internalUpdate.tokenSecretDef = '';
           } else {
@@ -190,6 +191,7 @@ async function patchFrameworkRequest(req, res, next) {
           'snapshot',
           'submissionTime',
           'configSecretDef',
+          'userSecretDef',
           'priorityClassDef',
           'dockerSecretDef',
           'tokenSecretDef',
@@ -206,6 +208,7 @@ async function patchFrameworkRequest(req, res, next) {
         snapshot.applyRequestPatch(patchData);
         const addOns = new AddOns(
           oldFramework.configSecretDef,
+          oldFramework.userSecretDef,
           oldFramework.priorityClassDef,
           oldFramework.dockerSecretDef,
           oldFramework.tokenSecretDef,
@@ -222,7 +225,7 @@ async function patchFrameworkRequest(req, res, next) {
 async function putFrameworkRequest(req, res, next) {
   // The handler to handle PUT /frameworkRequest.
   // PUT means provide a full spec of framework request, and the corresponding request will be created or updated.
-  // Along with the framework request, user must provide other job add-ons, e.g. configSecretDef, priorityClassDef, dockerSecretDef, tokenSecretDef.
+  // Along with the framework request, user must provide other job add-ons, e.g. configSecretDef, userSecretDef, priorityClassDef, dockerSecretDef, tokenSecretDef.
   // If the framework doesn't exist in database, the record will be created.
   // If the framework already exists, the record will be updated, and all job add-ons will be ignored. (Job add-ons can't be changed).
   // If the framework request JSON is changed(or created), we will mark it as requestSynced=false.
@@ -232,6 +235,7 @@ async function putFrameworkRequest(req, res, next) {
       frameworkRequest,
       submissionTime,
       configSecretDef,
+      userSecretDef,
       priorityClassDef,
       dockerSecretDef,
       tokenSecretDef,
@@ -254,6 +258,7 @@ async function putFrameworkRequest(req, res, next) {
           'snapshot',
           'submissionTime',
           'configSecretDef',
+          'userSecretDef',
           'priorityClassDef',
           'dockerSecretDef',
         ],
@@ -264,6 +269,7 @@ async function putFrameworkRequest(req, res, next) {
         // if the old framework doesn't exist, create add-ons.
         const addOns = new AddOns(
           configSecretDef,
+          userSecretDef,
           priorityClassDef,
           dockerSecretDef,
           tokenSecretDef,
@@ -275,6 +281,7 @@ async function putFrameworkRequest(req, res, next) {
         const oldSnapshot = new Snapshot(oldFramework.snapshot);
         const addOns = new AddOns(
           oldFramework.configSecretDef,
+          oldFramework.userSecretDef,
           oldFramework.priorityClassDef,
           oldFramework.dockerSecretDef,
           oldFramework.tokenSecretDef,

src/rest-server/src/models/v2/job/k8s.js

@@ -686,6 +686,7 @@ const generateTaskRole = (
 const generateFrameworkDescription = (
   frameworkName,
   virtualCluster,
+  userExtension,
   config,
   rawConfig,
 ) => {
@@ -770,6 +771,23 @@ const generateFrameworkDescription = (
         mountPath: '/usr/local/pai/secrets',
       });
     }
+    // mount user secrets to initContainers & job container if exist
+    if (userExtension) {
+      taskRoleDescription.task.pod.spec.volumes.push({
+        name: 'user-secrets',
+        secret: {
+          secretName: `${encodeName(frameworkName)}-usercred`,
+        },
+      });
+      taskRoleDescription.task.pod.spec.initContainers[0].volumeMounts.push({
+        name: 'user-secrets',
+        mountPath: '/usr/local/pai/secrets',
+      });
+      taskRoleDescription.task.pod.spec.containers[0].volumeMounts.push({
+        name: 'user-secrets',
+        mountPath: '/usr/local/pai/secrets',
+      });
+    }
     // mount token-secrets to initContainers & job container
     taskRoleDescription.task.pod.spec.volumes.push({
       name: 'token-secrets',
@@ -850,6 +868,22 @@ const getConfigSecretDef = (frameworkName, secrets) => {
   };
 };
 
+const getUserSecretDef = (frameworkName, secrets) => {
+  const data = {
+    'userSecrets.yaml': Buffer.from(yaml.safeDump(secrets)).toString('base64'),
+  };
+  return {
+    apiVersion: 'v1',
+    kind: 'Secret',
+    metadata: {
+      name: `${encodeName(frameworkName)}-usercred`,
+      namespace: 'default',
+    },
+    data: data,
+    type: 'Opaque',
+  };
+};
+
 const getTokenSecretDef = (frameworkName, token) => {
   const data = {
     token: Buffer.from(token).toString('base64'),
@@ -1083,9 +1117,17 @@ const put = async (frameworkName, config, rawConfig) => {
     }
   }
 
+  // generate the user-extension-secret definition
+  const user = await userModel.getUser(userName);
+  const userExtension = user.extension;
+  const userSecretDef = userExtension
+    ? getUserSecretDef(frameworkName, userExtension)
+    : null;
+
   const frameworkDescription = generateFrameworkDescription(
     frameworkName,
     virtualCluster,
+    userExtension,
     config,
     rawConfig,
   );
@@ -1149,6 +1191,7 @@ const put = async (frameworkName, config, rawConfig) => {
         frameworkRequest: frameworkDescription,
         submissionTime: submissionTime,
         configSecretDef: configSecretDef,
+        userSecretDef: userSecretDef,
         priorityClassDef: priorityClassDef,
         dockerSecretDef: dockerSecretDef,
         tokenSecretDef: tokenSecretDef,