feat: set/edit secrets for data connectors (#3330)

client/src/features/dataConnectorsV2/components/DataConnectorCredentialsModal.tsx

@@ -24,7 +24,7 @@ import { XLg } from "react-bootstrap-icons";
 import { RtkErrorAlert } from "../../../components/errors/RtkErrorAlert";
 import {
   useDeleteDataConnectorsByDataConnectorIdSecretsMutation,
-  usePostDataConnectorsByDataConnectorIdSecretsMutation,
+  usePatchDataConnectorsByDataConnectorIdSecretsMutation,
 } from "../../projectsV2/api/data-connectors.enhanced-api";
 import type { DataConnectorRead } from "../../projectsV2/api/data-connectors.api";
 import DataConnectorSecretsModal from "../../sessionsV2/DataConnectorSecretsModal";
@@ -49,7 +49,7 @@ export default function DataSourceCredentialsModal({
   });
 
   const [saveCredentials, saveCredentialsResult] =
-    usePostDataConnectorsByDataConnectorIdSecretsMutation();
+    usePatchDataConnectorsByDataConnectorIdSecretsMutation();
   const [deleteCredentials, deleteCredentialsResult] =
     useDeleteDataConnectorsByDataConnectorIdSecretsMutation();
 
@@ -65,7 +65,7 @@ export default function DataSourceCredentialsModal({
       const config = configs[0];
       saveCredentials({
         dataConnectorId: dataConnector.id,
-        cloudStorageSecretPostList: Object.entries(
+        dataConnectorSecretPatchList: Object.entries(
           config.sensitiveFieldValues
         ).map(([key, value]) => ({
           name: key,
@@ -88,15 +88,14 @@ export default function DataSourceCredentialsModal({
     }
   }, [deleteCredentialsResult, saveCredentialsResult.isSuccess, setOpen]);
   if (!isOpen) return null;
-
   if (
     dataConnector.storage.sensitive_fields == null ||
     dataConnector.storage.sensitive_fields.length === 0
   ) {
     return (
       <Modal
         centered
-        data-cy="cloud-storage-credentials-not-needed-modal"
+        data-cy="data-connector-credentials-not-needed-modal"
         isOpen={isOpen}
         size="lg"
       >
@@ -129,12 +128,12 @@ export default function DataSourceCredentialsModal({
     return (
       <Modal
         centered
-        data-cy="cloud-storage-credentials-error-modal"
+        data-cy="data-connector-credentials-error-modal"
         isOpen={isOpen}
         size="lg"
       >
         <ModalHeader className={cx("fw-bold")}>
-          Cloud Storage Credentials Update Error
+          Data Connector Credentials Update Error
         </ModalHeader>
         <ModalBody>
           <RtkErrorAlert error={error} />
@@ -156,12 +155,12 @@ export default function DataSourceCredentialsModal({
     return (
       <Modal
         centered
-        data-cy="cloud-storage-credentials-modal"
+        data-cy="data-connector-credentials-modal"
         isOpen={isOpen}
         size="lg"
       >
         <ModalHeader className={cx("fw-bold")}>
-          Saving Cloud Storage Credentials
+          Saving Data Connector Credentials
         </ModalHeader>
         <ModalBody>
           <Loader />
@@ -174,12 +173,12 @@ export default function DataSourceCredentialsModal({
     return (
       <Modal
         centered
-        data-cy="cloud-storage-credentials-modal"
+        data-cy="data-connector-credentials-modal"
         isOpen={isOpen}
         size="lg"
       >
         <ModalHeader className={cx("fw-bold")}>
-          Clearing Cloud Storage Credentials
+          Clearing Data Connector Credentials
         </ModalHeader>
         <ModalBody>
           <Loader />

client/src/features/dataConnectorsV2/components/DataConnectorModal/DataConnectorModalBody.tsx

@@ -288,7 +288,7 @@ export function DataConnectorMount({
   );
 
   return (
-    <form className="form-rk-green" data-cy="cloud-storage-edit-mount">
+    <form className="form-rk-green" data-cy="data-connector-edit-mount">
       <h5>Final details</h5>
       <p>We need a few more details to mount your data properly.</p>
 

client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx

@@ -46,9 +46,10 @@ import {
   hasProviderShortlist,
 } from "../../../project/utils/projectCloudStorage.utils";
 
-import { usePostStoragesV2ByStorageIdSecretsMutation } from "../../../projectsV2/api/projectV2.enhanced-api";
 import {
+  useGetDataConnectorsByDataConnectorIdSecretsQuery,
   usePatchDataConnectorsByDataConnectorIdMutation,
+  usePatchDataConnectorsByDataConnectorIdSecretsMutation,
   usePostDataConnectorsMutation,
 } from "../../../projectsV2/api/data-connectors.enhanced-api";
 import type { DataConnectorRead } from "../../../projectsV2/api/data-connectors.api";
@@ -87,6 +88,10 @@ export default function DataConnectorModal({
     isOpen ? undefined : skipToken
   );
   const { data: schema } = schemaQueryResult;
+  const { data: connectorSecrets } =
+    useGetDataConnectorsByDataConnectorIdSecretsQuery(
+      dataConnectorId ? { dataConnectorId } : skipToken
+    );
 
   // Reset state on props change
   useEffect(() => {
@@ -169,7 +174,7 @@ export default function DataConnectorModal({
   const [updateDataConnector, updateResult] =
     usePatchDataConnectorsByDataConnectorIdMutation();
   const [saveCredentials, saveCredentialsResult] =
-    usePostStoragesV2ByStorageIdSecretsMutation();
+    usePatchDataConnectorsByDataConnectorIdSecretsMutation();
   const [validateCloudStorageConnection, validationResult] =
     useTestCloudStorageConnectionMutation();
 
@@ -240,7 +245,6 @@ export default function DataConnectorModal({
         }
       });
     }
-
     validateCloudStorageConnection(validateParameters);
   }, [flatDataConnector, validateCloudStorageConnection]);
 
@@ -323,7 +327,7 @@ export default function DataConnectorModal({
     const sensitiveFieldNames = findSensitive(
       schema.find((s) => s.prefix === flatDataConnector.schema)
     );
-    const cloudStorageSecretPostList = sensitiveFieldNames
+    const dataConnectorSecretPatchList = sensitiveFieldNames
       .map((name) => ({
         name,
         value: options[name],
@@ -334,8 +338,8 @@ export default function DataConnectorModal({
         value: "" + secret.value,
       }));
     saveCredentials({
-      storageId: dataConnectorId,
-      cloudStorageSecretPostList,
+      dataConnectorId,
+      dataConnectorSecretPatchList,
     });
   }, [
     createResult.data?.id,
@@ -398,31 +402,28 @@ export default function DataConnectorModal({
     : "Please go back and select a provider";
   const isResultLoading = isAddResultLoading || isModifyResultLoading;
 
-  const storageSecrets =
-    dataConnector != null && "secrets" in dataConnector
-      ? dataConnector.secrets ?? []
-      : [];
-  const hasStoredCredentialsInConfig = storageSecrets.length > 0;
+  const hasStoredCredentialsInConfig =
+    connectorSecrets != null && connectorSecrets.length > 0;
 
   return (
     <Modal
       backdrop="static"
       centered
       className={styles.modal}
-      data-cy="cloud-storage-edit-modal"
+      data-cy="data-connector-edit-modal"
       fullscreen="lg"
-      id={dataConnector?.id ?? "new-cloud-storage"}
+      id={dataConnector?.id ?? "new-data-connector"}
       isOpen={isOpen}
       scrollable
       size="lg"
       unmountOnClose={false}
       toggle={toggle}
     >
-      <ModalHeader toggle={toggle} data-cy="cloud-storage-edit-header">
+      <ModalHeader toggle={toggle} data-cy="data-connector-edit-header">
         <DataConnectorModalHeader dataConnectorId={dataConnectorId} />
       </ModalHeader>
 
-      <ModalBody data-cy="cloud-storage-edit-body">
+      <ModalBody data-cy="data-connector-edit-body">
         <DataConnectorModalBody
           dataConnectorResultName={dataConnectorResultName}
           flatDataConnector={flatDataConnector}
@@ -432,13 +433,13 @@ export default function DataConnectorModal({
           setStateSafe={setStateSafe}
           setFlatDataConnectorSafe={setFlatDataConnectorSafe}
           state={state}
-          storageSecrets={storageSecrets}
+          storageSecrets={connectorSecrets ?? []}
           success={success}
           validationSucceeded={validationSucceeded}
         />
       </ModalBody>
 
-      <ModalFooter className="border-top" data-cy="cloud-storage-edit-footer">
+      <ModalFooter className="border-top" data-cy="data-connector-edit-footer">
         <DataConnectorConnectionTestResult
           validationResult={validationResult}
         />
@@ -453,7 +454,7 @@ export default function DataConnectorModal({
         {!isResultLoading && !success && (
           <Button
             color="outline-danger"
-            data-cy="cloud-storage-edit-rest-button"
+            data-cy="data-connector-edit-rest-button"
             disabled={validationResult.isLoading}
             onClick={() => {
               reset();

client/src/features/dataConnectorsV2/components/DataConnectorView.tsx

@@ -23,6 +23,7 @@ import { CredentialMoreInfo } from "../../project/components/cloudStorage/CloudS
 import { CLOUD_STORAGE_SAVED_SECRET_DISPLAY_VALUE } from "../../project/components/cloudStorage/projectCloudStorage.constants";
 import { getCredentialFieldDefinitions } from "../../project/utils/projectCloudStorage.utils";
 import type { DataConnectorRead } from "../../projectsV2/api/data-connectors.api";
+import { useGetDataConnectorsByDataConnectorIdSecretsQuery } from "../../projectsV2/api/data-connectors.enhanced-api";
 import { storageSecretNameToFieldName } from "../../secrets/secrets.utils";
 import DataConnectorActions from "./DataConnectorActions";
 
@@ -36,21 +37,28 @@ export default function DataConnectorView({
   showView,
   toggleView,
 }: DataConnectorViewProps) {
+  const { data: connectorSecrets } =
+    useGetDataConnectorsByDataConnectorIdSecretsQuery({
+      dataConnectorId: dataConnector.id,
+    });
   const storageDefinition = dataConnector.storage;
   const sensitiveFields = storageDefinition.sensitive_fields
     ? storageDefinition.sensitive_fields?.map((f) => f.name)
     : [];
   const anySensitiveField = Object.keys(storageDefinition.configuration).some(
     (key) => sensitiveFields.includes(key)
   );
-  const storageSecrets = dataConnector.secrets;
   const savedCredentialFields =
-    storageSecrets?.reduce((acc: Record<string, string>, s) => {
+    connectorSecrets?.reduce((acc: Record<string, string>, s) => {
       acc[storageSecretNameToFieldName(s)] = s.name;
       return acc;
     }, {}) ?? {};
   const credentialFieldDefinitions = useMemo(
-    () => getCredentialFieldDefinitions(dataConnector),
+    () =>
+      getCredentialFieldDefinitions({
+        storage: dataConnector.storage,
+        sensitive_fields: dataConnector.storage.sensitive_fields,
+      }),
     [dataConnector]
   );
   const requiredCredentials = useMemo(
@@ -75,15 +83,15 @@ export default function DataConnectorView({
           <button
             aria-label="Close"
             className="btn-close"
-            data-cy="data-source-view-back-button"
+            data-cy="data-connector-view-back-button"
             data-bs-dismiss="offcanvas"
             onClick={toggleView}
           ></button>
         </div>
 
         <div className="mb-4">
           <div className={cx("d-flex", "justify-content-between")}>
-            <h2 className="m-0" data-cy="data-source-title">
+            <h2 className="m-0" data-cy="data-connector-title">
               {dataConnector.name}
             </h2>
             <div className="my-auto">

client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx

@@ -351,7 +351,9 @@ function DataConnectorDisplay({ dataConnector }: DataConnectorDisplayProps) {
       >
         <Row className={cx("align-items-center", "g-2")}>
           <Col>
-            <span className="fw-bold">{name}</span>
+            <span className="fw-bold" data-cy="data-connector-name">
+              {name}
+            </span>
             {description && <ClampedParagraph>{description}</ClampedParagraph>}
             <div
               className={cx(

client/src/features/dataConnectorsV2/components/dataConnector.utils.ts

@@ -158,15 +158,15 @@ export function dataConnectorToFlattened(
 export function validationParametersFromDataConnectorConfiguration(
   config: DataConnectorConfiguration
 ) {
-  const newStorageDetails = dataConnectorToFlattened(
-    _dataConnectorFromConfig(config)
-  );
-  return _validationConfigurationFromDataConnectorFlat(newStorageDetails);
+  const dataConnector = _dataConnectorFromConfig(config);
+  const validateParameters: TestCloudStorageConnectionParams = {
+    configuration: dataConnector.configuration,
+    source_path: dataConnector.source_path,
+  };
+  return validateParameters;
 }
 
-function _dataConnectorFromConfig(
-  config: DataConnectorConfiguration
-): DataConnectorRead {
+function _dataConnectorFromConfig(config: DataConnectorConfiguration) {
   const dataConnector = config.dataConnector;
   const storageDefinition = config.dataConnector.storage;
   const mergedDataConnector = { ...dataConnector, ...storageDefinition };
@@ -181,27 +181,3 @@ function _dataConnectorFromConfig(
   });
   return mergedDataConnector;
 }
-
-function _validationConfigurationFromDataConnectorFlat(
-  dataConnector: DataConnectorFlat
-) {
-  const validateParameters: TestCloudStorageConnectionParams = {
-    configuration: {
-      type: dataConnector.schema,
-    },
-    source_path: dataConnector.sourcePath ?? "/",
-  };
-  if (dataConnector.provider) {
-    validateParameters.configuration.provider = dataConnector.provider;
-  }
-  if (dataConnector.options && Object.keys(dataConnector.options).length > 0) {
-    const options = dataConnector.options as CloudStorageDetailsOptions;
-    Object.entries(options).forEach(([key, value]) => {
-      if (value != undefined && value !== "") {
-        validateParameters.configuration[key] = value;
-      }
-    });
-  }
-
-  return validateParameters;
-}

client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts

@@ -20,10 +20,8 @@ import { useMemo } from "react";
 
 import { CLOUD_OPTIONS_OVERRIDE } from "../../project/components/cloudStorage/projectCloudStorage.constants";
 import { RCloneOption } from "../../projectsV2/api/data-connectors.api";
-import type {
-  DataConnectorRead,
-  CloudStorageSecretGet,
-} from "../../projectsV2/api/data-connectors.api";
+import type { DataConnectorRead } from "../../projectsV2/api/data-connectors.api";
+import { useGetDataConnectorsListSecretsQuery } from "../../projectsV2/api/data-connectors.enhanced-api";
 
 import type { SessionStartCloudStorageConfiguration } from "../../sessionsV2/startSessionOptionsV2.types";
 
@@ -39,6 +37,9 @@ interface UseDataSourceConfigurationArgs {
 export default function useDataConnectorConfiguration({
   dataConnectors,
 }: UseDataSourceConfigurationArgs) {
+  const { data: dataConnectorSecrets } = useGetDataConnectorsListSecretsQuery({
+    dataConnectorIds: dataConnectors?.map((dc) => dc.id) ?? [],
+  });
   const dataConnectorConfigs = useMemo(
     () =>
       dataConnectors?.map((dataConnector) => {
@@ -77,15 +78,8 @@ export default function useDataConnectorConfiguration({
           if (name == null) return;
           sensitiveFieldValues[name] = "";
         });
-        const storagesSecrets = dataConnectors?.reduce(
-          (a: Record<string, CloudStorageSecretGet[]>, s) => {
-            a[s.id] = s.secrets ? s.secrets : [];
-            return a;
-          },
-          {}
-        );
-        const savedCredentialFields = storagesSecrets
-          ? storagesSecrets[dataConnector.id].map((s) => s.name)
+        const savedCredentialFields = dataConnectorSecrets
+          ? dataConnectorSecrets[dataConnector.id].map((s) => s.name)
           : [];
         return {
           active: true,
@@ -96,7 +90,7 @@ export default function useDataConnectorConfiguration({
           savedCredentialFields,
         };
       }),
-    [dataConnectors]
+    [dataConnectors, dataConnectorSecrets]
   );
 
   return {

client/src/features/groupsV2/show/GroupV2Show.tsx

@@ -26,7 +26,7 @@ import {
   useNavigate,
   useParams,
 } from "react-router-dom-v5-compat";
-import { Badge, Col, Row } from "reactstrap";
+import { Col, Row } from "reactstrap";
 
 import { Loader } from "../../../components/Loader";
 import ContainerWrap from "../../../components/container/ContainerWrap";