From 663415a0d986519c20310816032a7a344704702a Mon Sep 17 00:00:00 2001 From: Chandrasekhar Ramakrishnan Date: Tue, 24 Sep 2024 11:30:55 +0200 Subject: [PATCH] feat: support secrets on data connectors --- .../DataConnectorCredentialsModal.tsx | 21 +- .../DataConnectorModalBody.tsx | 2 +- .../components/DataConnectorModal/index.tsx | 37 +- .../components/DataConnectorView.tsx | 18 +- .../components/DataConnectorsBox.tsx | 4 +- .../components/dataConnector.utils.ts | 38 +- .../useDataConnectorConfiguration.hook.ts | 22 +- .../projectsV2/api/data-connectors.api.ts | 108 +++- .../api/data-connectors.enhanced-api.ts | 47 +- .../api/data-connectors.openapi.json | 224 ++++++-- .../sessionsV2/DataConnectorSecretsModal.tsx | 24 +- tests/cypress/e2e/groupV2.spec.ts | 6 +- .../e2e/groupV2DataSourceCredentials.spec.ts | 491 ++++++++++++++++++ .../data-connector-secrets-partial.json | 2 +- ...a-connector-with-secrets-values-empty.json | 44 -- ...ta-connector-with-secrets-values-full.json | 53 -- ...connector-with-secrets-values-partial.json | 49 -- .../data-connector-with-secrets.json | 43 -- .../dataConnector/data-connector.json | 68 ++- .../renkulab-fixtures/dataConnectors.ts | 70 +++ 20 files changed, 980 insertions(+), 391 deletions(-) create mode 100644 tests/cypress/e2e/groupV2DataSourceCredentials.spec.ts delete mode 100644 tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json delete mode 100644 tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json delete mode 100644 tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json delete mode 100644 tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json diff --git a/client/src/features/dataConnectorsV2/components/DataConnectorCredentialsModal.tsx b/client/src/features/dataConnectorsV2/components/DataConnectorCredentialsModal.tsx index 3c578cc7f..ceb8db1e2 100644 --- a/client/src/features/dataConnectorsV2/components/DataConnectorCredentialsModal.tsx +++ b/client/src/features/dataConnectorsV2/components/DataConnectorCredentialsModal.tsx @@ -24,7 +24,7 @@ import { XLg } from "react-bootstrap-icons"; import { RtkErrorAlert } from "../../../components/errors/RtkErrorAlert"; import { useDeleteDataConnectorsByDataConnectorIdSecretsMutation, - usePostDataConnectorsByDataConnectorIdSecretsMutation, + usePatchDataConnectorsByDataConnectorIdSecretsMutation, } from "../../projectsV2/api/data-connectors.enhanced-api"; import type { DataConnectorRead } from "../../projectsV2/api/data-connectors.api"; import DataConnectorSecretsModal from "../../sessionsV2/DataConnectorSecretsModal"; @@ -49,7 +49,7 @@ export default function DataSourceCredentialsModal({ }); const [saveCredentials, saveCredentialsResult] = - usePostDataConnectorsByDataConnectorIdSecretsMutation(); + usePatchDataConnectorsByDataConnectorIdSecretsMutation(); const [deleteCredentials, deleteCredentialsResult] = useDeleteDataConnectorsByDataConnectorIdSecretsMutation(); @@ -65,7 +65,7 @@ export default function DataSourceCredentialsModal({ const config = configs[0]; saveCredentials({ dataConnectorId: dataConnector.id, - cloudStorageSecretPostList: Object.entries( + dataConnectorSecretPatchList: Object.entries( config.sensitiveFieldValues ).map(([key, value]) => ({ name: key, @@ -88,7 +88,6 @@ export default function DataSourceCredentialsModal({ } }, [deleteCredentialsResult, saveCredentialsResult.isSuccess, setOpen]); if (!isOpen) return null; - if ( dataConnector.storage.sensitive_fields == null || dataConnector.storage.sensitive_fields.length === 0 @@ -96,7 +95,7 @@ export default function DataSourceCredentialsModal({ return ( @@ -129,12 +128,12 @@ export default function DataSourceCredentialsModal({ return ( - Cloud Storage Credentials Update Error + Data Connector Credentials Update Error @@ -156,12 +155,12 @@ export default function DataSourceCredentialsModal({ return ( - Saving Cloud Storage Credentials + Saving Data Connector Credentials @@ -174,12 +173,12 @@ export default function DataSourceCredentialsModal({ return ( - Clearing Cloud Storage Credentials + Clearing Data Connector Credentials diff --git a/client/src/features/dataConnectorsV2/components/DataConnectorModal/DataConnectorModalBody.tsx b/client/src/features/dataConnectorsV2/components/DataConnectorModal/DataConnectorModalBody.tsx index 88f2226b5..238049ec5 100644 --- a/client/src/features/dataConnectorsV2/components/DataConnectorModal/DataConnectorModalBody.tsx +++ b/client/src/features/dataConnectorsV2/components/DataConnectorModal/DataConnectorModalBody.tsx @@ -288,7 +288,7 @@ export function DataConnectorMount({ ); return ( -
+
Final details

We need a few more details to mount your data properly.

diff --git a/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx b/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx index 79db65af6..cf8db09e5 100644 --- a/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx +++ b/client/src/features/dataConnectorsV2/components/DataConnectorModal/index.tsx @@ -46,9 +46,10 @@ import { hasProviderShortlist, } from "../../../project/utils/projectCloudStorage.utils"; -import { usePostStoragesV2ByStorageIdSecretsMutation } from "../../../projectsV2/api/projectV2.enhanced-api"; import { + useGetDataConnectorsByDataConnectorIdSecretsQuery, usePatchDataConnectorsByDataConnectorIdMutation, + usePatchDataConnectorsByDataConnectorIdSecretsMutation, usePostDataConnectorsMutation, } from "../../../projectsV2/api/data-connectors.enhanced-api"; import type { DataConnectorRead } from "../../../projectsV2/api/data-connectors.api"; @@ -87,6 +88,10 @@ export default function DataConnectorModal({ isOpen ? undefined : skipToken ); const { data: schema } = schemaQueryResult; + const { data: connectorSecrets } = + useGetDataConnectorsByDataConnectorIdSecretsQuery( + dataConnectorId ? { dataConnectorId } : skipToken + ); // Reset state on props change useEffect(() => { @@ -169,7 +174,7 @@ export default function DataConnectorModal({ const [updateDataConnector, updateResult] = usePatchDataConnectorsByDataConnectorIdMutation(); const [saveCredentials, saveCredentialsResult] = - usePostStoragesV2ByStorageIdSecretsMutation(); + usePatchDataConnectorsByDataConnectorIdSecretsMutation(); const [validateCloudStorageConnection, validationResult] = useTestCloudStorageConnectionMutation(); @@ -240,7 +245,6 @@ export default function DataConnectorModal({ } }); } - validateCloudStorageConnection(validateParameters); }, [flatDataConnector, validateCloudStorageConnection]); @@ -323,7 +327,7 @@ export default function DataConnectorModal({ const sensitiveFieldNames = findSensitive( schema.find((s) => s.prefix === flatDataConnector.schema) ); - const cloudStorageSecretPostList = sensitiveFieldNames + const dataConnectorSecretPatchList = sensitiveFieldNames .map((name) => ({ name, value: options[name], @@ -334,8 +338,8 @@ export default function DataConnectorModal({ value: "" + secret.value, })); saveCredentials({ - storageId: dataConnectorId, - cloudStorageSecretPostList, + dataConnectorId, + dataConnectorSecretPatchList, }); }, [ createResult.data?.id, @@ -398,31 +402,28 @@ export default function DataConnectorModal({ : "Please go back and select a provider"; const isResultLoading = isAddResultLoading || isModifyResultLoading; - const storageSecrets = - dataConnector != null && "secrets" in dataConnector - ? dataConnector.secrets ?? [] - : []; - const hasStoredCredentialsInConfig = storageSecrets.length > 0; + const hasStoredCredentialsInConfig = + connectorSecrets != null && connectorSecrets.length > 0; return ( - + - + - + @@ -453,7 +454,7 @@ export default function DataConnectorModal({ {!isResultLoading && !success && ( @@ -83,7 +91,7 @@ export default function DataConnectorView({
-

+

{dataConnector.name}

diff --git a/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx b/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx index 992df2e71..6c596551e 100644 --- a/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx +++ b/client/src/features/dataConnectorsV2/components/DataConnectorsBox.tsx @@ -351,7 +351,9 @@ function DataConnectorDisplay({ dataConnector }: DataConnectorDisplayProps) { > - {name} + + {name} + {description && {description}}
0) { - const options = dataConnector.options as CloudStorageDetailsOptions; - Object.entries(options).forEach(([key, value]) => { - if (value != undefined && value !== "") { - validateParameters.configuration[key] = value; - } - }); - } - - return validateParameters; -} diff --git a/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts b/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts index ad30d866f..4f7d6531e 100644 --- a/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts +++ b/client/src/features/dataConnectorsV2/components/useDataConnectorConfiguration.hook.ts @@ -20,10 +20,8 @@ import { useMemo } from "react"; import { CLOUD_OPTIONS_OVERRIDE } from "../../project/components/cloudStorage/projectCloudStorage.constants"; import { RCloneOption } from "../../projectsV2/api/data-connectors.api"; -import type { - DataConnectorRead, - CloudStorageSecretGet, -} from "../../projectsV2/api/data-connectors.api"; +import type { DataConnectorRead } from "../../projectsV2/api/data-connectors.api"; +import { useGetDataConnectorsListSecretsQuery } from "../../projectsV2/api/data-connectors.enhanced-api"; import type { SessionStartCloudStorageConfiguration } from "../../sessionsV2/startSessionOptionsV2.types"; @@ -39,6 +37,9 @@ interface UseDataSourceConfigurationArgs { export default function useDataConnectorConfiguration({ dataConnectors, }: UseDataSourceConfigurationArgs) { + const { data: dataConnectorSecrets } = useGetDataConnectorsListSecretsQuery({ + dataConnectorIds: dataConnectors?.map((dc) => dc.id) ?? [], + }); const dataConnectorConfigs = useMemo( () => dataConnectors?.map((dataConnector) => { @@ -77,15 +78,8 @@ export default function useDataConnectorConfiguration({ if (name == null) return; sensitiveFieldValues[name] = ""; }); - const storagesSecrets = dataConnectors?.reduce( - (a: Record, s) => { - a[s.id] = s.secrets ? s.secrets : []; - return a; - }, - {} - ); - const savedCredentialFields = storagesSecrets - ? storagesSecrets[dataConnector.id].map((s) => s.name) + const savedCredentialFields = dataConnectorSecrets + ? dataConnectorSecrets[dataConnector.id].map((s) => s.name) : []; return { active: true, @@ -96,7 +90,7 @@ export default function useDataConnectorConfiguration({ savedCredentialFields, }; }), - [dataConnectors] + [dataConnectors, dataConnectorSecrets] ); return { diff --git a/client/src/features/projectsV2/api/data-connectors.api.ts b/client/src/features/projectsV2/api/data-connectors.api.ts index b52b7353d..1cf7c5698 100644 --- a/client/src/features/projectsV2/api/data-connectors.api.ts +++ b/client/src/features/projectsV2/api/data-connectors.api.ts @@ -56,6 +56,33 @@ const injectedRtkApi = api.injectEndpoints({ url: `/namespaces/${queryArg["namespace"]}/data_connectors/${queryArg.slug}`, }), }), + getDataConnectorsByDataConnectorIdProjectLinks: build.query< + GetDataConnectorsByDataConnectorIdProjectLinksApiResponse, + GetDataConnectorsByDataConnectorIdProjectLinksApiArg + >({ + query: (queryArg) => ({ + url: `/data_connectors/${queryArg.dataConnectorId}/project_links`, + }), + }), + postDataConnectorsByDataConnectorIdProjectLinks: build.mutation< + PostDataConnectorsByDataConnectorIdProjectLinksApiResponse, + PostDataConnectorsByDataConnectorIdProjectLinksApiArg + >({ + query: (queryArg) => ({ + url: `/data_connectors/${queryArg.dataConnectorId}/project_links`, + method: "POST", + body: queryArg.dataConnectorToProjectLinkPost, + }), + }), + deleteDataConnectorsByDataConnectorIdProjectLinksAndLinkId: build.mutation< + DeleteDataConnectorsByDataConnectorIdProjectLinksAndLinkIdApiResponse, + DeleteDataConnectorsByDataConnectorIdProjectLinksAndLinkIdApiArg + >({ + query: (queryArg) => ({ + url: `/data_connectors/${queryArg.dataConnectorId}/project_links/${queryArg.linkId}`, + method: "DELETE", + }), + }), getDataConnectorsByDataConnectorIdSecrets: build.query< GetDataConnectorsByDataConnectorIdSecretsApiResponse, GetDataConnectorsByDataConnectorIdSecretsApiArg @@ -64,14 +91,14 @@ const injectedRtkApi = api.injectEndpoints({ url: `/data_connectors/${queryArg.dataConnectorId}/secrets`, }), }), - postDataConnectorsByDataConnectorIdSecrets: build.mutation< - PostDataConnectorsByDataConnectorIdSecretsApiResponse, - PostDataConnectorsByDataConnectorIdSecretsApiArg + patchDataConnectorsByDataConnectorIdSecrets: build.mutation< + PatchDataConnectorsByDataConnectorIdSecretsApiResponse, + PatchDataConnectorsByDataConnectorIdSecretsApiArg >({ query: (queryArg) => ({ url: `/data_connectors/${queryArg.dataConnectorId}/secrets`, - method: "POST", - body: queryArg.cloudStorageSecretPostList, + method: "PATCH", + body: queryArg.dataConnectorSecretPatchList, }), }), deleteDataConnectorsByDataConnectorIdSecrets: build.mutation< @@ -120,23 +147,44 @@ export type DeleteDataConnectorsByDataConnectorIdApiArg = { dataConnectorId: Ulid; }; export type GetNamespacesByNamespaceDataConnectorsAndSlugApiResponse = - /** status 200 The data connectors */ DataConnectorRead; + /** status 200 The data connector */ DataConnectorRead; export type GetNamespacesByNamespaceDataConnectorsAndSlugApiArg = { namespace: string; slug: string; }; +export type GetDataConnectorsByDataConnectorIdProjectLinksApiResponse = + /** status 200 List of data connector to project links */ DataConnectorToProjectLinksList; +export type GetDataConnectorsByDataConnectorIdProjectLinksApiArg = { + /** the ID of the data connector */ + dataConnectorId: Ulid; +}; +export type PostDataConnectorsByDataConnectorIdProjectLinksApiResponse = + /** status 201 The data connector was connected to a project */ DataConnectorToProjectLink; +export type PostDataConnectorsByDataConnectorIdProjectLinksApiArg = { + /** the ID of the data connector */ + dataConnectorId: Ulid; + dataConnectorToProjectLinkPost: DataConnectorToProjectLinkPost; +}; +export type DeleteDataConnectorsByDataConnectorIdProjectLinksAndLinkIdApiResponse = + /** status 204 The data connector was removed or did not exist in the first place */ void; +export type DeleteDataConnectorsByDataConnectorIdProjectLinksAndLinkIdApiArg = { + /** the ID of the data connector */ + dataConnectorId: Ulid; + /** the ID of the link between a data connector and a project */ + linkId: Ulid; +}; export type GetDataConnectorsByDataConnectorIdSecretsApiResponse = - /** status 200 The saved storage secrets */ CloudStorageSecretGetList; + /** status 200 The saved storage secrets */ DataConnectorSecretsList; export type GetDataConnectorsByDataConnectorIdSecretsApiArg = { /** the ID of the data connector */ dataConnectorId: Ulid; }; -export type PostDataConnectorsByDataConnectorIdSecretsApiResponse = - /** status 201 The secrets for cloud storage were saved */ CloudStorageSecretGetList; -export type PostDataConnectorsByDataConnectorIdSecretsApiArg = { +export type PatchDataConnectorsByDataConnectorIdSecretsApiResponse = + /** status 201 The secrets for cloud storage were saved */ DataConnectorSecretsList; +export type PatchDataConnectorsByDataConnectorIdSecretsApiArg = { /** the ID of the data connector */ dataConnectorId: Ulid; - cloudStorageSecretPostList: CloudStorageSecretPostList; + dataConnectorSecretPatchList: DataConnectorSecretPatchList; }; export type DeleteDataConnectorsByDataConnectorIdSecretsApiResponse = /** status 204 The secrets were removed or did not exist in the first place or the storage doesn't exist */ void; @@ -205,9 +253,8 @@ export type CloudStorageCoreRead = { readonly: StorageReadOnly; sensitive_fields: RCloneOption[]; }; -export type CloudStorageSecretGet = { - /** Name of the field to store credential for */ - name: string; +export type DataConnectorSecret = { + name: DataConnectorName; secret_id: Ulid; }; export type CreationDate = string; @@ -223,7 +270,7 @@ export type DataConnector = { namespace: Slug; slug: Slug; storage: CloudStorageCore; - secrets?: CloudStorageSecretGet[]; + secrets?: DataConnectorSecret[]; creation_date: CreationDate; created_by: UserId; visibility: Visibility; @@ -237,7 +284,7 @@ export type DataConnectorRead = { namespace: Slug; slug: Slug; storage: CloudStorageCoreRead; - secrets?: CloudStorageSecretGet[]; + secrets?: DataConnectorSecret[]; creation_date: CreationDate; created_by: UserId; visibility: Visibility; @@ -333,14 +380,24 @@ export type DataConnectorPatchRead = { description?: Description; keywords?: KeywordsList; }; -export type CloudStorageSecretGetList = CloudStorageSecretGet[]; -export type SecretValue = string; -export type CloudStorageSecretPost = { - /** Name of the field to store credential for */ - name: string; - value: SecretValue; +export type DataConnectorToProjectLink = { + id: Ulid; + data_connector_id: Ulid; + project_id: Ulid; + creation_date: CreationDate; + created_by: UserId; +}; +export type DataConnectorToProjectLinksList = DataConnectorToProjectLink[]; +export type DataConnectorToProjectLinkPost = { + project_id: Ulid; +}; +export type DataConnectorSecretsList = DataConnectorSecret[]; +export type SecretValueNullable = string | null; +export type DataConnectorSecretPatch = { + name: DataConnectorName; + value: SecretValueNullable; }; -export type CloudStorageSecretPostList = CloudStorageSecretPost[]; +export type DataConnectorSecretPatchList = DataConnectorSecretPatch[]; export const { useGetDataConnectorsQuery, usePostDataConnectorsMutation, @@ -348,7 +405,10 @@ export const { usePatchDataConnectorsByDataConnectorIdMutation, useDeleteDataConnectorsByDataConnectorIdMutation, useGetNamespacesByNamespaceDataConnectorsAndSlugQuery, + useGetDataConnectorsByDataConnectorIdProjectLinksQuery, + usePostDataConnectorsByDataConnectorIdProjectLinksMutation, + useDeleteDataConnectorsByDataConnectorIdProjectLinksAndLinkIdMutation, useGetDataConnectorsByDataConnectorIdSecretsQuery, - usePostDataConnectorsByDataConnectorIdSecretsMutation, + usePatchDataConnectorsByDataConnectorIdSecretsMutation, useDeleteDataConnectorsByDataConnectorIdSecretsMutation, } = injectedRtkApi; diff --git a/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts b/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts index 90ca0ae5a..e846b4fa6 100644 --- a/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts +++ b/client/src/features/projectsV2/api/data-connectors.enhanced-api.ts @@ -6,6 +6,8 @@ import { dataConnectorsApi as api } from "./data-connectors.api"; import type { GetDataConnectorsApiArg, GetDataConnectorsApiResponse as GetDataConnectorsApiResponseOrig, + GetDataConnectorsByDataConnectorIdSecretsApiArg, + GetDataConnectorsByDataConnectorIdSecretsApiResponse, } from "./data-connectors.api"; export interface GetDataConnectorsApiResponse @@ -13,6 +15,15 @@ export interface GetDataConnectorsApiResponse dataConnectors: GetDataConnectorsApiResponseOrig; } +interface GetDataConnectorListSecretsApiArg { + dataConnectorIds: GetDataConnectorsByDataConnectorIdSecretsApiArg["dataConnectorId"][]; +} + +type GetDataConnectorListSecretsApiResponse = Record< + string, + GetDataConnectorsByDataConnectorIdSecretsApiResponse +>; + const injectedApi = api.injectEndpoints({ endpoints: (builder) => ({ getDataConnectorsPaged: builder.query< @@ -47,6 +58,26 @@ const injectedApi = api.injectEndpoints({ }; }, }), + getDataConnectorsListSecrets: builder.query< + GetDataConnectorListSecretsApiResponse, + GetDataConnectorListSecretsApiArg + >({ + async queryFn(queryArg, _api, _options, fetchWithBQ) { + const { dataConnectorIds } = queryArg; + const result: GetDataConnectorListSecretsApiResponse = {}; + for (const dataConnectorId of dataConnectorIds) { + const response = await fetchWithBQ( + `/data_connectors/${dataConnectorId}/secrets` + ); + if (response.error) { + return response; + } + result[dataConnectorId] = + response.data as GetDataConnectorsByDataConnectorIdSecretsApiResponse; + } + return { data: result }; + }, + }), }), }); @@ -62,15 +93,21 @@ const enhancedApi = injectedApi.enhanceEndpoints({ getDataConnectorsPaged: { providesTags: ["DataConnectors"], }, + getDataConnectorsListSecrets: { + providesTags: ["DataConnectorSecrets"], + }, + getDataConnectorsByDataConnectorIdSecrets: { + providesTags: ["DataConnectorSecrets"], + }, patchDataConnectorsByDataConnectorId: { invalidatesTags: ["DataConnectors"], }, + patchDataConnectorsByDataConnectorIdSecrets: { + invalidatesTags: ["DataConnectorSecrets"], + }, postDataConnectors: { invalidatesTags: ["DataConnectors"], }, - postDataConnectorsByDataConnectorIdSecrets: { - invalidatesTags: ["DataConnectorSecrets"], - }, }, }); @@ -80,7 +117,9 @@ export const { useDeleteDataConnectorsByDataConnectorIdMutation, useDeleteDataConnectorsByDataConnectorIdSecretsMutation, useGetDataConnectorsPagedQuery: useGetDataConnectorsQuery, + useGetDataConnectorsByDataConnectorIdSecretsQuery, + useGetDataConnectorsListSecretsQuery, usePatchDataConnectorsByDataConnectorIdMutation, usePostDataConnectorsMutation, - usePostDataConnectorsByDataConnectorIdSecretsMutation, + usePatchDataConnectorsByDataConnectorIdSecretsMutation, } = enhancedApi; diff --git a/client/src/features/projectsV2/api/data-connectors.openapi.json b/client/src/features/projectsV2/api/data-connectors.openapi.json index 2366677f8..b1b7cdc0a 100644 --- a/client/src/features/projectsV2/api/data-connectors.openapi.json +++ b/client/src/features/projectsV2/api/data-connectors.openapi.json @@ -227,7 +227,7 @@ "summary": "Get a data connector by namespace and project slug", "responses": { "200": { - "description": "The data connectors", + "description": "The data connector", "content": { "application/json": { "schema": { @@ -253,6 +253,101 @@ "tags": ["data connectors"] } }, + "/data_connectors/{data_connector_id}/project_links": { + "parameters": [ + { + "in": "path", + "name": "data_connector_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ulid" + }, + "description": "the ID of the data connector" + } + ], + "get": { + "summary": "Get all links from a given data connector to projects", + "responses": { + "200": { + "description": "List of data connector to project links", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataConnectorToProjectLinksList" + } + } + } + }, + "default": { + "$ref": "#/components/responses/Error" + } + }, + "tags": ["data connectors"] + }, + "post": { + "summary": "Create a new link from a data connector to a project", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataConnectorToProjectLinkPost" + } + } + } + }, + "responses": { + "201": { + "description": "The data connector was connected to a project", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DataConnectorToProjectLink" + } + } + } + }, + "default": { + "$ref": "#/components/responses/Error" + } + }, + "tags": ["data connectors"] + } + }, + "/data_connectors/{data_connector_id}/project_links/{link_id}": { + "parameters": [ + { + "in": "path", + "name": "data_connector_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ulid" + }, + "description": "the ID of the data connector" + }, + { + "in": "path", + "name": "link_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/Ulid" + }, + "description": "the ID of the link between a data connector and a project" + } + ], + "delete": { + "summary": "Remove a link from a data connector to a project", + "responses": { + "204": { + "description": "The data connector was removed or did not exist in the first place" + }, + "default": { + "$ref": "#/components/responses/Error" + } + }, + "tags": ["data connectors"] + } + }, "/data_connectors/{data_connector_id}/secrets": { "parameters": [ { @@ -273,7 +368,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CloudStorageSecretGetList" + "$ref": "#/components/schemas/DataConnectorSecretsList" } } } @@ -294,14 +389,15 @@ }, "tags": ["data connectors"] }, - "post": { + "patch": { "summary": "Save secrets for a data connector", + "description": "New secrets will be added and existing secrets will have their value updated. Using `null` as a value will remove the corresponding secret.", "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CloudStorageSecretPostList" + "$ref": "#/components/schemas/DataConnectorSecretPatchList" } } } @@ -312,7 +408,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CloudStorageSecretGetList" + "$ref": "#/components/schemas/DataConnectorSecretsList" } } } @@ -369,7 +465,7 @@ "secrets": { "type": "array", "items": { - "$ref": "#/components/schemas/CloudStorageSecretGet" + "$ref": "#/components/schemas/DataConnectorSecret" } }, "creation_date": { @@ -588,45 +684,66 @@ "storage_url": "s3://giab" } }, - "CloudStorageSecretPost": { + "DataConnectorToProjectLinksList": { + "description": "A list of links from a data connector to a project", + "type": "array", + "items": { + "$ref": "#/components/schemas/DataConnectorToProjectLink" + } + }, + "DataConnectorToProjectLink": { + "description": "A link from a data connector to a project in Renku 2.0", "type": "object", - "description": "Data for storing secret for a storage field", + "additionalProperties": false, "properties": { - "name": { - "type": "string", - "description": "Name of the field to store credential for", - "minLength": 1, - "maxLength": 99 + "id": { + "$ref": "#/components/schemas/Ulid" }, - "value": { - "$ref": "#/components/schemas/SecretValue" + "data_connector_id": { + "$ref": "#/components/schemas/Ulid" + }, + "project_id": { + "$ref": "#/components/schemas/Ulid" + }, + "creation_date": { + "$ref": "#/components/schemas/CreationDate" + }, + "created_by": { + "$ref": "#/components/schemas/UserId" } }, - "required": ["name", "value"] + "required": [ + "id", + "data_connector_id", + "project_id", + "creation_date", + "created_by" + ] }, - "CloudStorageSecretPostList": { - "description": "List of storage secrets that are saved", - "type": "array", - "items": { - "$ref": "#/components/schemas/CloudStorageSecretPost" - } + "DataConnectorToProjectLinkPost": { + "description": "A link to be created from a data connector to a project in Renku 2.0", + "type": "object", + "additionalProperties": false, + "properties": { + "project_id": { + "$ref": "#/components/schemas/Ulid" + } + }, + "required": ["project_id"] }, - "CloudStorageSecretGetList": { - "description": "List of storage secrets that are saved", + "DataConnectorSecretsList": { + "description": "A list of data connectors", "type": "array", "items": { - "$ref": "#/components/schemas/CloudStorageSecretGet" + "$ref": "#/components/schemas/DataConnectorSecret" } }, - "CloudStorageSecretGet": { + "DataConnectorSecret": { + "description": "Information about a credential saved for a data connector", "type": "object", - "description": "Data for saved storage secrets", "properties": { "name": { - "type": "string", - "description": "Name of the field to store credential for", - "minLength": 1, - "maxLength": 99 + "$ref": "#/components/schemas/DataConnectorName" }, "secret_id": { "$ref": "#/components/schemas/Ulid" @@ -634,36 +751,31 @@ }, "required": ["name", "secret_id"] }, - "SecretValue": { - "description": "Secret value that can be any text", - "type": "string", - "minLength": 1, - "maxLength": 5000 + "DataConnectorSecretPatchList": { + "description": "List of secrets to be saved for a data connector", + "type": "array", + "items": { + "$ref": "#/components/schemas/DataConnectorSecretPatch" + } }, - "RCloneEntry": { - "type": "object", - "description": "Schema for a storage type in rclone, like S3 or Azure Blob Storage. Contains fields for that storage type.", + "DataConnectorSecretPatch": { + "description": "Information about a credential to save for a data connector", "properties": { "name": { - "type": "string", - "description": "Human readable name of the provider" - }, - "description": { - "type": "string", - "description": "description of the provider" - }, - "prefix": { - "type": "string", - "description": "Machine readable name of the provider" + "$ref": "#/components/schemas/DataConnectorName" }, - "options": { - "description": "Fields/properties used for this storage.", - "type": "array", - "items": { - "$ref": "#/components/schemas/RCloneOption" - } + "value": { + "$ref": "#/components/schemas/SecretValueNullable" } - } + }, + "required": ["name", "value"] + }, + "SecretValueNullable": { + "description": "Secret value that can be any text", + "type": "string", + "minLength": 1, + "maxLength": 5000, + "nullable": true }, "RCloneOption": { "type": "object", diff --git a/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx b/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx index a6c075957..4a0088033 100644 --- a/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx +++ b/client/src/features/sessionsV2/DataConnectorSecretsModal.tsx @@ -56,17 +56,17 @@ import { validationParametersFromDataConnectorConfiguration } from "../dataConne const CONTEXT_STRINGS = { session: { continueButton: "Continue", - dataCy: "session-cloud-storage-credentials-modal", + dataCy: "session-data-connector-credentials-modal", header: "Session Storage Credentials", testError: - "The data source could not be mounted. Please retry with different credentials, or skip the test. If you skip, the data source will still try to mount, using the provided credentials, at session launch time.", + "The data connector could not be mounted. Please retry with different credentials, or skip the test. If you skip, the data source will still try to mount, using the provided credentials, at session launch time.", }, storage: { continueButton: "Test and Save", - dataCy: "cloud-storage-credentials-modal", - header: "Cloud Storage Credentials", + dataCy: "data-connector-credentials-modal", + header: "Data Connector Credentials", testError: - "The data source could not be mounted. Please try different credentials or rely on providing credentials at session launch time.", + "The data connector could not be mounted. Please try different credentials or rely on providing credentials at session launch time.", }, }; @@ -164,21 +164,21 @@ export default function DataConnectorSecretsModal({ isOpen, onCancel, onStart, - dataConnectorConfigs: initialCloudStorageConfigs, + dataConnectorConfigs: initialDataConnectorConfigs, }: DataConnectorSecretsModalProps) { const noCredentialsConfigs = useMemo( () => - initialCloudStorageConfigs == null + initialDataConnectorConfigs == null ? [] - : initialCloudStorageConfigs.filter( + : initialDataConnectorConfigs.filter( (config) => config.sensitiveFieldDefinitions.length === 0 ), - [initialCloudStorageConfigs] + [initialDataConnectorConfigs] ); const [dataConnectorConfigs, setDataConnectorConfigs] = useState( - initialCloudStorageConfigs == null + initialDataConnectorConfigs == null ? [] - : initialCloudStorageConfigs.filter( + : initialDataConnectorConfigs.filter( (config) => config.sensitiveFieldDefinitions.length > 0 ) ); @@ -280,7 +280,7 @@ export default function DataConnectorSecretsModal({ diff --git a/tests/cypress/e2e/groupV2.spec.ts b/tests/cypress/e2e/groupV2.spec.ts index e6ee1b0e9..e1d871e61 100644 --- a/tests/cypress/e2e/groupV2.spec.ts +++ b/tests/cypress/e2e/groupV2.spec.ts @@ -268,12 +268,12 @@ describe("Work with group data connectors", () => { cy.get("#secret_access_key").type("secret key"); cy.getDataCy("test-data-connector-button").click(); cy.getDataCy("add-data-connector-continue-button").contains("Skip").click(); - cy.getDataCy("cloud-storage-edit-mount").within(() => { + cy.getDataCy("data-connector-edit-mount").within(() => { cy.get("#name").type("example storage without credentials"); }); cy.getDataCy("data-connector-edit-update-button").click(); cy.wait("@postDataConnector"); - cy.getDataCy("cloud-storage-edit-body").should( + cy.getDataCy("data-connector-edit-body").should( "contain.text", "The data connector test-2-group-v2/example-storage-without-credentials has been successfully added." ); @@ -302,7 +302,7 @@ describe("Work with group data connectors", () => { .click(); cy.getDataCy("data-connector-edit-update-button").click(); cy.wait("@patchDataConnector"); - cy.getDataCy("cloud-storage-edit-body").should( + cy.getDataCy("data-connector-edit-body").should( "contain.text", "The data connector test-2-group-v2/public-storage has been successfully updated." ); diff --git a/tests/cypress/e2e/groupV2DataSourceCredentials.spec.ts b/tests/cypress/e2e/groupV2DataSourceCredentials.spec.ts new file mode 100644 index 000000000..b38468baa --- /dev/null +++ b/tests/cypress/e2e/groupV2DataSourceCredentials.spec.ts @@ -0,0 +1,491 @@ +/*! + * Copyright 2024 - Swiss Data Science Center (SDSC) + * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and + * Eidgenössische Technische Hochschule Zürich (ETHZ). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fixtures from "../support/renkulab-fixtures"; + +function openDataSourceMenu() { + cy.getDataCy("data-connector-edit") + .parent() + .find("[data-cy=button-with-menu-dropdown]") + .first() + .click(); +} + +describe("Set up data sources with credentials", () => { + beforeEach(() => { + fixtures + .config() + .versions() + .userTest() + .dataServicesUser({ + response: { id: "0945f006-e117-49b7-8966-4c0842146313" }, + }) + .namespaces() + .readGroupV2() + .readGroupV2Namespace() + .listGroupV2Members() + .listProjectV2ByNamespace() + .readGroupV2() + .readGroupV2Namespace() + .listGroupV2Members() + .listProjectV2ByNamespace(); + fixtures.projects().landingUserProjects().listGroupV2(); + }); + + it("shows information about credentials", () => { + fixtures + .listDataConnectors({ namespace: "test-2-group-v2" }) + .dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets-empty.json", + }); + cy.visit("/v2/groups/test-2-group-v2"); + cy.wait("@readGroupV2"); + // add data connector + cy.getDataCy("data-connector-name").contains("private-storage-1").click(); + cy.getDataCy("data-connector-title").should( + "contain.text", + "private-storage-1" + ); + cy.getDataCy("access_key_id-value").should("contain.text", ""); + cy.getDataCy("data-connector-view-back-button").click(); + }); + + it("create data connector after failed connection test", () => { + fixtures + .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" }) + .listDataConnectors({ namespace: "test-2-group-v2" }) + .testCloudStorage({ success: false }) + .postDataConnector({ namespace: "test-2-group-v2" }) + .dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets-empty.json", + }) + .patchDataConnectorSecrets({ + content: [], + // No call to postCloudStorageSecrets is expected + shouldNotBeCalled: true, + }); + cy.visit("/v2/groups/test-2-group-v2"); + cy.wait("@readGroupV2"); + // add data connector + cy.getDataCy("add-data-connector").should("be.visible").click(); + cy.wait("@getStorageSchema"); + + // Pick a provider + cy.getDataCy("data-storage-s3").click(); + cy.getDataCy("data-provider-AWS").click(); + cy.getDataCy("data-connector-edit-next-button").click(); + + // Fill out the details + cy.get("#sourcePath").type("bucket/my-source"); + cy.get("#access_key_id").type("access key"); + cy.get("#secret_access_key").type("secret key"); + cy.getDataCy("test-data-connector-button").click(); + cy.getDataCy("add-data-connector-continue-button").contains("Skip").click(); + cy.getDataCy("data-connector-edit-mount").within(() => { + cy.get("#name").type("example storage without credentials"); + }); + cy.getDataCy("data-connector-edit-update-button").click(); + cy.wait("@postDataConnector"); + cy.getDataCy("data-connector-edit-body").should( + "contain.text", + "The data connector test-2-group-v2/example-storage-without-credentials has been successfully added." + ); + cy.getDataCy("data-connector-edit-close-button").click(); + cy.wait("@getDataConnectors"); + }); + + it("create data connector with credentials", () => { + fixtures + .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" }) + .listDataConnectors({ namespace: "test-2-group-v2" }) + .testCloudStorage({ success: true }) + .postDataConnector({ namespace: "test-2-group-v2" }) + .patchDataConnectorSecrets({ + dataConnectorId: "ULID-5", + content: [ + { + name: "access_key_id", + value: "access key", + }, + { + name: "secret_access_key", + value: "secret key", + }, + ], + }); + cy.visit("/v2/groups/test-2-group-v2"); + cy.wait("@readGroupV2"); + // add data connector + cy.getDataCy("add-data-connector").should("be.visible").click(); + cy.wait("@getStorageSchema"); + + // Pick a provider + cy.getDataCy("data-storage-s3").click(); + cy.getDataCy("data-provider-AWS").click(); + cy.getDataCy("data-connector-edit-next-button").click(); + + // Fill out the details + cy.get("#sourcePath").type("bucket/my-source"); + cy.get("#access_key_id").type("access key"); + cy.get("#secret_access_key").type("secret key"); + cy.getDataCy("test-data-connector-button").click(); + cy.getDataCy("add-data-connector-continue-button") + .contains("Continue") + .click(); + + cy.getDataCy("data-connector-edit-mount").within(() => { + cy.get("#name").type("example storage"); + cy.get("#saveCredentials").should("be.checked"); + }); + cy.getDataCy("data-connector-edit-update-button").click(); + fixtures.dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets.json", + name: "getDataConnectorSecrets", + }); + cy.wait("@postDataConnector"); + cy.wait("@patchDataConnectorSecrets"); + cy.getDataCy("data-connector-edit-body").should( + "contain.text", + "The data connector test-2-group-v2/example-storage has been successfully added, along with its credentials." + ); + cy.getDataCy("data-connector-edit-close-button").click(); + cy.wait("@getDataConnectors"); + }); + + it("set credentials for a data connector", () => { + fixtures + .listDataConnectors({ + fixture: "dataConnector/data-connector.json", + namespace: "test-2-group-v2", + }) + .dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets-empty.json", + }); + + cy.visit("/v2/groups/test-2-group-v2"); + cy.wait("@readGroupV2"); + cy.wait("@getDataConnectors"); + // Credentials should not yet be stored + cy.getDataCy("data-connector-name").contains("example storage").click(); + cy.wait("@getDataConnectorSecrets"); + cy.getDataCy("data-connector-title").should( + "contain.text", + "example storage" + ); + cy.getDataCy("access_key_id-value").should("contain.text", ""); + + // set credentials + openDataSourceMenu(); + cy.getDataCy("data-connector-credentials").click(); + + fixtures + .testCloudStorage({ success: true }) + .patchDataConnectorSecrets({ + dataConnectorId: "ULID-1", + content: [ + { + name: "access_key_id", + value: "access key", + }, + { + name: "secret_access_key", + value: "secret key", + }, + ], + }) + .dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets.json", + }); + + cy.getDataCy("data-connector-credentials-modal") + .find("#access_key_id") + .type("access key"); + cy.getDataCy("data-connector-credentials-modal") + .find("#secret_access_key") + .type("secret key"); + cy.getDataCy("data-connector-credentials-modal") + .contains("Test and Save") + .click(); + + cy.wait("@testCloudStorage"); + cy.wait("@patchDataConnectorSecrets"); + cy.wait("@getDataConnectorSecrets"); + + // Credentials should be stored + cy.getDataCy("data-connector-title").should( + "contain.text", + "example storage" + ); + cy.getDataCy("access_key_id-value").should( + "contain.text", + "" + ); + + // edit data source, without touching the credentials + fixtures.getStorageSchema({ + fixture: "cloudStorage/storage-schema-s3.json", + }); + openDataSourceMenu(); + cy.getDataCy("data-connector-edit").click(); + cy.getDataCy("data-connector-edit-modal") + .find("#access_key_id") + .invoke("attr", "value") + .should("eq", ""); + cy.getDataCy("data-connector-edit-modal") + .find("#secret_access_key") + .invoke("attr", "value") + .should("eq", ""); + }); + + it("clear credentials for a data connector", () => { + fixtures + .listDataConnectors({ + fixture: "dataConnector/data-connector.json", + namespace: "test-2-group-v2", + }) + .dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets-partial.json", + name: "getDataConnectorSecrets", + }); + cy.visit("/v2/groups/test-2-group-v2"); + cy.wait("@readGroupV2"); + cy.wait("@getDataConnectors"); + + // Credentials should be stored + cy.getDataCy("data-connector-name").contains("example storage").click(); + cy.wait("@getDataConnectorSecrets"); + cy.getDataCy("data-connector-title").should( + "contain.text", + "example storage" + ); + cy.getDataCy("access_key_id-value").should( + "contain.text", + "" + ); + + // clear credentials + openDataSourceMenu(); + cy.getDataCy("data-connector-credentials").click(); + cy.getDataCy("data-connector-credentials-modal") + .contains("The saved credentials for this data source are incomplete") + .should("be.visible"); + + fixtures.deleteDataConnectorSecrets().dataConnectorSecrets({ + fixture: "dataConnector/data-connector-secrets-empty.json", + name: "getDataConnectorSecrets2", + }); + cy.getDataCy("data-connector-credentials-modal").contains("Clear").click(); + cy.wait("@deleteDataConnectorSecrets"); + cy.wait("@getDataConnectorSecrets2"); + + // Credentials should be changed + cy.getDataCy("data-connector-title").should( + "contain.text", + "example storage" + ); + cy.getDataCy("access_key_id-value").should("contain.text", ""); + }); + + // it("edit a data source with credentials", () => { + // fixtures.testCloudStorage(); + // fixtures + // .testCloudStorage() + // .sessionServersEmpty() + // .sessionImage() + // .resourcePoolsTest() + // .cloudStorage({ + // isV2: true, + // fixture: "cloudStorage/cloud-storage-with-secrets-values-partial.json", + // name: "getCloudStorageV2", + // }) + // .cloudStorageSecrets({ + // fixture: "cloudStorage/cloud-storage-secrets-partial.json", + // }) + // .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" }) + // .postCloudStorage({ + // name: "postCloudStorageV2", + // fixture: "cloudStorage/new-cloud-storage_v2.json", + // }) + // .sessionLaunchers({ + // fixture: "projectV2/session-launchers.json", + // }); + + // cy.visit("/v2/projects/user1-uuid/test-2-v2-project"); + // cy.wait("@readProjectV2"); + // cy.wait("@getSessionServers"); + // cy.wait("@sessionLaunchers"); + // // Credentials should be stored + // cy.getDataCy("data-storage-name").should("contain.text", "example-storage"); + // cy.getDataCy("data-storage-name").click(); + // cy.getDataCy("data-source-title").should("contain.text", "example-storage"); + // cy.getDataCy("secret_access_key-value").should( + // "contain.text", + // "" + // ); + // cy.getDataCy("data-source-view-back-button").click(); + + // // edit data source, without touching the credentials + // openDataSourceMenu(); + // cy.getDataCy("data-source-edit").click(); + // cy.getDataCy("cloud-storage-edit-modal") + // .find("#access_key_id") + // .invoke("attr", "value") + // .should("eq", ""); + // cy.getDataCy("cloud-storage-edit-modal") + // .find("#secret_access_key") + // .invoke("attr", "value") + // .should("eq", ""); + // cy.getDataCy("cloud-storage-edit-modal").contains("Next").click(); + + // fixtures.patchCloudStorage({ name: "patchCloudStorage", isV2: true }); + // cy.getDataCy("cloud-storage-edit-modal").contains("Update storage").click(); + // cy.wait("@patchCloudStorage"); + // }); + + // describe("Set up multiple data sources", () => { + // beforeEach(() => { + // fixtures + // .config() + // .versions() + // .userTest() + // .namespaces() + // .dataServicesUser({ + // response: { + // id: "0945f006-e117-49b7-8966-4c0842146313", + // email: "user1@email.com", + // }, + // }) + // .listProjectV2Members(); + // fixtures.projects().landingUserProjects().readProjectV2(); + // }); + + // it("set up one data source that succeeds, another with failed credentials", () => { + // fixtures + // .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" }) + // .postCloudStorage({ + // name: "postCloudStorageV2", + // fixture: "cloudStorage/new-cloud-storage_v2.json", + // }) + // .cloudStorage({ + // isV2: true, + // fixture: "cloudStorage/cloud-storage-with-secrets-values-full.json", + // name: "getCloudStorageV2", + // }) + // .postCloudStorageSecrets({ + // content: [ + // { + // name: "access_key_id", + // value: "access key", + // }, + // { + // name: "secret_access_key", + // value: "secret key", + // }, + // ], + // }) + // .testCloudStorage({ success: true }); + // cy.visit("/v2/projects/user1-uuid/test-2-v2-project"); + // cy.wait("@readProjectV2"); + // // add data connector + // cy.getDataCy("add-data-source").click(); + // cy.wait("@getStorageSchema"); + // cy.getDataCy("data-storage-s3").click(); + // cy.getDataCy("data-provider-AWS").click(); + // cy.getDataCy("cloud-storage-edit-next-button").click(); + // cy.get("#sourcePath").type("bucket/my-source"); + // cy.get("#access_key_id").type("access key"); + // cy.get("#secret_access_key").type("secret key"); + // cy.getDataCy("test-cloud-storage-button").click(); + // cy.getDataCy("add-cloud-storage-continue-button") + // .contains("Continue") + // .click(); + // cy.getDataCy("cloud-storage-edit-mount").within(() => { + // cy.get("#name").type("example-storage"); + // cy.get("#saveCredentials").should("be.checked"); + // }); + // cy.getDataCy("cloud-storage-edit-update-button").click(); + // fixtures.cloudStorageSecrets({ + // fixture: "cloudStorage/cloud-storage-secrets.json", + // name: "getCloudStorageSecrets2", + // }); + // cy.wait("@postCloudStorageV2"); + // cy.wait("@postCloudStorageSecrets"); + // cy.getDataCy("cloud-storage-edit-body").should( + // "contain.text", + // "The storage example-storage has been successfully added, along with its credentials." + // ); + // cy.getDataCy("cloud-storage-edit-close-button").click(); + // cy.wait("@getCloudStorageV2"); + // cy.getDataCy("data-storage-name").should("contain.text", "example-storage"); + // cy.getDataCy("data-storage-name").click(); + // cy.getDataCy("data-source-title").should("contain.text", "example-storage"); + // cy.getDataCy("access_key_id-value").should( + // "contain.text", + // "" + // ); + // cy.getDataCy("data-source-view-back-button").click(); + + // fixtures + // .getStorageSchema({ fixture: "cloudStorage/storage-schema-s3.json" }) + // .postCloudStorage({ + // name: "postCloudStorageV2", + // fixture: "cloudStorage/new-cloud-storage_v2.json", + // }) + // .cloudStorage({ + // isV2: true, + // fixture: "cloudStorage/cloud-storage-with-secrets-values-empty.json", + // name: "getCloudStorageV2", + // }) + // .cloudStorageSecrets({ + // fixture: "cloudStorage/cloud-storage-secrets-empty.json", + // }) + // .testCloudStorage({ success: false }) + // .postCloudStorageSecrets({ + // content: [], + // // No call to postCloudStorageSecrets is expected + // shouldNotBeCalled: true, + // }); + // // add data connector + // cy.getDataCy("add-data-source").click(); + // cy.getDataCy("data-storage-s3").click(); + // cy.getDataCy("data-provider-AWS").click(); + // cy.getDataCy("cloud-storage-edit-next-button").click(); + // cy.get("#sourcePath").type("bucket/my-source"); + // cy.get("#access_key_id").type("access key"); + // cy.get("#secret_access_key").type("secret key"); + // cy.getDataCy("test-cloud-storage-button").click(); + // cy.getDataCy("add-cloud-storage-continue-button").contains("Skip").click(); + // cy.getDataCy("cloud-storage-edit-mount").within(() => { + // cy.get("#name").type("example-storage-no-credentials"); + // }); + // cy.getDataCy("cloud-storage-edit-update-button").click(); + // cy.wait("@postCloudStorageV2"); + // cy.getDataCy("cloud-storage-edit-body").should( + // "contain.text", + // "The storage example-storage has been successfully added." + // ); + // cy.getDataCy("cloud-storage-edit-close-button").click(); + // cy.wait("@getCloudStorageV2"); + + // cy.getDataCy("data-storage-name").should("contain.text", "example-storage"); + // cy.getDataCy("data-storage-name").click(); + // cy.getDataCy("data-source-title").should("contain.text", "example-storage"); + // cy.getDataCy("access_key_id-value").should("contain.text", ""); + // cy.getDataCy("data-source-view-back-button").click(); + // }); +}); diff --git a/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json b/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json index b74beb578..c84af1120 100644 --- a/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json +++ b/tests/cypress/fixtures/dataConnector/data-connector-secrets-partial.json @@ -1,6 +1,6 @@ [ { - "name": "secret_access_key", + "name": "access_key_id", "secret_id": "ULID1" } ] diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json deleted file mode 100644 index 387f54769..000000000 --- a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-empty.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "storage": { - "configuration": { - "type": "s3", - "provider": "AWS", - "access_key_id": "", - "secret_access_key": "" - }, - "name": "example-storage", - "project_id": 1, - "readonly": true, - "source_path": "bucket/my-source", - "storage_id": "2", - "storage_type": "s3", - "target_path": "external_storage/aws" - }, - "sensitive_fields": [ - { - "name": "access_key_id", - "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - }, - { - "name": "secret_access_key", - "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - } - ], - "secrets": [] - } -] diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json deleted file mode 100644 index b2b4e0561..000000000 --- a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-full.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "storage": { - "configuration": { - "type": "s3", - "provider": "AWS", - "access_key_id": "", - "secret_access_key": "" - }, - "name": "example-storage", - "project_id": 1, - "readonly": true, - "source_path": "bucket/my-source", - "storage_id": "2", - "storage_type": "s3", - "target_path": "external_storage/aws" - }, - "sensitive_fields": [ - { - "name": "access_key_id", - "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - }, - { - "name": "secret_access_key", - "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - } - ], - "secrets": [ - { - "name": "access_key_id", - "secret_id": "ULID2" - }, - { - "name": "secret_access_key", - "secret_id": "ULID1" - } - ] - } -] diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json deleted file mode 100644 index be3c74eef..000000000 --- a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets-values-partial.json +++ /dev/null @@ -1,49 +0,0 @@ -[ - { - "storage": { - "configuration": { - "type": "s3", - "provider": "AWS", - "access_key_id": "", - "secret_access_key": "" - }, - "name": "example-storage", - "project_id": 1, - "readonly": true, - "source_path": "bucket/my-source", - "storage_id": "2", - "storage_type": "s3", - "target_path": "external_storage/aws" - }, - "sensitive_fields": [ - { - "name": "access_key_id", - "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - }, - { - "name": "secret_access_key", - "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - } - ], - "secrets": [ - { - "name": "secret_access_key", - "secret_id": "ULID1" - } - ] - } -] diff --git a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json b/tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json deleted file mode 100644 index 8c16dfcd7..000000000 --- a/tests/cypress/fixtures/dataConnector/data-connector-with-secrets.json +++ /dev/null @@ -1,43 +0,0 @@ -[ - { - "storage": { - "configuration": { - "type": "s3", - "provider": "AWS", - "access_key_id": "", - "secret_access_key": "" - }, - "name": "example-storage", - "project_id": 1, - "readonly": true, - "source_path": "bucket/my-source", - "storage_id": "2", - "storage_type": "s3", - "target_path": "external_storage/aws" - }, - "sensitive_fields": [ - { - "name": "access_key_id", - "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - }, - { - "name": "secret_access_key", - "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.", - "provider": "", - "default": "", - "default_str": "", - "required": false, - "sensitive": true, - "advanced": false, - "exclusive": false - } - ] - } -] diff --git a/tests/cypress/fixtures/dataConnector/data-connector.json b/tests/cypress/fixtures/dataConnector/data-connector.json index 0e2548b72..5398bd208 100644 --- a/tests/cypress/fixtures/dataConnector/data-connector.json +++ b/tests/cypress/fixtures/dataConnector/data-connector.json @@ -1,22 +1,48 @@ -{ - "id": "ULID-1", - "name": "example-storage", - "namespace": "user1-uuid", - "slug": "example-storage", - "storage": { - "storage_type": "s3", - "configuration": { - "type": "s3", - "provider": "Other", - "endpoint": "https://s3.example.com" +[ + { + "id": "ULID-1", + "name": "example storage", + "namespace": "user1-uuid", + "slug": "example-storage", + "storage": { + "storage_type": "s3", + "configuration": { + "type": "s3", + "provider": "AWS", + "access_key_id": "", + "secret_access_key": "" + }, + "source_path": "bucket/my-source", + "target_path": "external_storage/aws", + "readonly": true, + "sensitive_fields": [ + { + "name": "access_key_id", + "help": "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.", + "provider": "", + "default": "", + "default_str": "", + "required": false, + "sensitive": true, + "advanced": false, + "exclusive": false + }, + { + "name": "secret_access_key", + "help": "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.", + "provider": "", + "default": "", + "default_str": "", + "required": false, + "sensitive": true, + "advanced": false, + "exclusive": false + } + ] }, - "source_path": "bucket/source", - "target_path": "mount/path", - "readonly": true, - "sensitive_fields": [] - }, - "creation_date": "2023-11-15T09:55:59Z", - "created_by": { "id": "user1-uuid" }, - "visibility": "public", - "description": "Data connector 1 description" -} + "creation_date": "2023-11-15T09:55:59Z", + "created_by": { "id": "user1-uuid" }, + "visibility": "public", + "description": "Example storage description" + } +] diff --git a/tests/cypress/support/renkulab-fixtures/dataConnectors.ts b/tests/cypress/support/renkulab-fixtures/dataConnectors.ts index 65927428e..0ce2ecc0b 100644 --- a/tests/cypress/support/renkulab-fixtures/dataConnectors.ts +++ b/tests/cypress/support/renkulab-fixtures/dataConnectors.ts @@ -28,6 +28,18 @@ interface DataConnectorArgs extends SimpleFixture { visibility?: string; } +interface DataConnectorSecretsArgs extends SimpleFixture { + dataConnectorId?: string; +} + +interface PatchDataConnectorSecretsArgs extends DataConnectorSecretsArgs { + content: { + name: string; + value: string; + }[]; + shouldNotBeCalled?: boolean; +} + export function DataConnector(Parent: T) { return class DataConnectorFixtures extends Parent { listDataConnectors(args?: DataConnectorArgs) { @@ -121,5 +133,63 @@ export function DataConnector(Parent: T) { ).as(name); return this; } + + dataConnectorSecrets(args?: DataConnectorSecretsArgs) { + const { + fixture = "tests/cypress/fixtures/dataConnector/data-connector-secrets.json", + name = "getDataConnectorSecrets", + dataConnectorId = "ULID-1", + } = args ?? {}; + const response = { fixture }; + cy.intercept( + "GET", + `/ui-server/api/data/data_connectors/${dataConnectorId}/secrets`, + response + ).as(name); + return this; + } + + deleteDataConnectorSecrets(args?: DataConnectorSecretsArgs) { + const { + name = "deleteDataConnectorSecrets", + dataConnectorId = "ULID-1", + } = args ?? {}; + // eslint-disable-next-line max-nested-callbacks + cy.intercept( + "DELETE", + `/ui-server/api/data/data_connectors/${dataConnectorId}/secrets`, + { body: null, delay: 1000 } + ).as(name); + return this; + } + + patchDataConnectorSecrets(args?: PatchDataConnectorSecretsArgs) { + const { + content, + fixture = "dataConnector/data-connector-secrets-empty.json", + name = "patchDataConnectorSecrets", + shouldNotBeCalled = false, + dataConnectorId = "ULID-5", + } = args ?? {}; + cy.fixture(fixture).then((secrets) => { + // eslint-disable-next-line max-nested-callbacks + cy.intercept( + "PATCH", + `/ui-server/api/data/data_connectors/${dataConnectorId}/secrets`, + (req) => { + if (shouldNotBeCalled) + throw new Error("No call to post secrets expected"); + const newSecrets = req.body; + expect(newSecrets.length).equal(content.length); + newSecrets.forEach((secret, index) => { + expect(secret.name).equal(content[index].name); + expect(secret.value).equal(content[index].value); + }); + req.reply({ body: secrets, delay: 1000 }); + } + ).as(name); + }); + return this; + } }; }