feat(data-warehouse): edit SQL based import configs (#25685)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

frontend/src/lib/api.ts

@@ -50,11 +50,11 @@ import {
     Experiment,
     ExportedAssetType,
     ExternalDataJob,
+    ExternalDataSource,
     ExternalDataSourceCreatePayload,
     ExternalDataSourceSchema,
     ExternalDataSourceSyncSchema,
     ExternalDataSourceType,
-    ExternalDataStripeSource,
     FeatureFlagAssociatedRoleType,
     FeatureFlagType,
     Group,
@@ -854,7 +854,7 @@ class ApiRequest {
         return this.projectsDetail(teamId).addPathComponent('external_data_sources')
     }
 
-    public externalDataSource(sourceId: ExternalDataStripeSource['id'], teamId?: TeamType['id']): ApiRequest {
+    public externalDataSource(sourceId: ExternalDataSource['id'], teamId?: TeamType['id']): ApiRequest {
         return this.externalDataSources(teamId).addPathComponent(sourceId)
     }
 
@@ -2199,25 +2199,25 @@ const api = {
         },
     },
     externalDataSources: {
-        async list(options?: ApiMethodOptions | undefined): Promise<PaginatedResponse<ExternalDataStripeSource>> {
+        async list(options?: ApiMethodOptions | undefined): Promise<PaginatedResponse<ExternalDataSource>> {
             return await new ApiRequest().externalDataSources().get(options)
         },
-        async get(sourceId: ExternalDataStripeSource['id']): Promise<ExternalDataStripeSource> {
+        async get(sourceId: ExternalDataSource['id']): Promise<ExternalDataSource> {
             return await new ApiRequest().externalDataSource(sourceId).get()
         },
         async create(data: Partial<ExternalDataSourceCreatePayload>): Promise<{ id: string }> {
             return await new ApiRequest().externalDataSources().create({ data })
         },
-        async delete(sourceId: ExternalDataStripeSource['id']): Promise<void> {
+        async delete(sourceId: ExternalDataSource['id']): Promise<void> {
             await new ApiRequest().externalDataSource(sourceId).delete()
         },
-        async reload(sourceId: ExternalDataStripeSource['id']): Promise<void> {
+        async reload(sourceId: ExternalDataSource['id']): Promise<void> {
             await new ApiRequest().externalDataSource(sourceId).withAction('reload').create()
         },
         async update(
-            sourceId: ExternalDataStripeSource['id'],
-            data: Partial<ExternalDataStripeSource>
-        ): Promise<ExternalDataStripeSource> {
+            sourceId: ExternalDataSource['id'],
+            data: Partial<ExternalDataSource>
+        ): Promise<ExternalDataSource> {
             return await new ApiRequest().externalDataSource(sourceId).update({ data })
         },
         async database_schema(
@@ -2239,7 +2239,7 @@ const api = {
                 .create({ data: { source_type, prefix } })
         },
         async jobs(
-            sourceId: ExternalDataStripeSource['id'],
+            sourceId: ExternalDataSource['id'],
             before: string | null,
             after: string | null
         ): Promise<ExternalDataJob[]> {

frontend/src/scenes/data-warehouse/external/forms/SourceForm.tsx

@@ -1,5 +1,4 @@
 import { LemonFileInput, LemonInput, LemonSelect, LemonSwitch, LemonTextArea } from '@posthog/lemon-ui'
-import { useValues } from 'kea'
 import { Form, Group } from 'kea-forms'
 import { LemonField } from 'lib/lemon-ui/LemonField'
 
@@ -8,22 +7,27 @@ import { SourceConfig, SourceFieldConfig } from '~/types'
 import { SOURCE_DETAILS, sourceWizardLogic } from '../../new/sourceWizardLogic'
 import { DataWarehouseIntegrationChoice } from './DataWarehouseIntegrationChoice'
 
-interface SourceFormProps {
+export interface SourceFormProps {
     sourceConfig: SourceConfig
     showPrefix?: boolean
-    showSourceFields?: boolean
+    jobInputs?: Record<string, any>
 }
 
-const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConfig): JSX.Element => {
+const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConfig, lastValue?: any): JSX.Element => {
     if (field.type === 'switch-group') {
         return (
             <LemonField key={field.name} name={[field.name, 'enabled']} label={field.label}>
                 {({ value, onChange }) => (
                     <>
-                        <LemonSwitch checked={value} onChange={onChange} />
+                        <LemonSwitch
+                            checked={value === undefined || value === null ? lastValue?.['enabled'] : value}
+                            onChange={onChange}
+                        />
                         {value && (
                             <Group name={field.name}>
-                                {field.fields.map((field) => sourceFieldToElement(field, sourceConfig))}
+                                {field.fields.map((field) =>
+                                    sourceFieldToElement(field, sourceConfig, lastValue?.[field.name])
+                                )}
                             </Group>
                         )}
                     </>
@@ -43,11 +47,21 @@ const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConf
             >
                 {({ value, onChange }) => (
                     <>
-                        <LemonSelect options={field.options} value={value ?? field.defaultValue} onChange={onChange} />
+                        <LemonSelect
+                            options={field.options}
+                            value={
+                                value === undefined || value === null
+                                    ? lastValue?.[field.name]
+                                    : value || field.defaultValue
+                            }
+                            onChange={onChange}
+                        />
                         <Group name={field.name}>
                             {field.options
                                 .find((n) => n.value === (value ?? field.defaultValue))
-                                ?.fields?.map((field) => sourceFieldToElement(field, sourceConfig))}
+                                ?.fields?.map((field) =>
+                                    sourceFieldToElement(field, sourceConfig, lastValue?.[field.name])
+                                )}
                         </Group>
                     </>
                 )}
@@ -63,6 +77,7 @@ const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConf
                     data-attr={field.name}
                     placeholder={field.placeholder}
                     minRows={4}
+                    defaultValue={lastValue}
                 />
             </LemonField>
         )
@@ -102,32 +117,33 @@ const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConf
                 data-attr={field.name}
                 placeholder={field.placeholder}
                 type={field.type}
+                defaultValue={lastValue}
             />
         </LemonField>
     )
 }
 
-export default function SourceForm({ sourceConfig }: SourceFormProps): JSX.Element {
-    const { source } = useValues(sourceWizardLogic)
-    const showSourceFields = SOURCE_DETAILS[sourceConfig.name].showSourceForm
-        ? SOURCE_DETAILS[sourceConfig.name].showSourceForm?.(source.payload)
-        : true
-    const showPrefix = SOURCE_DETAILS[sourceConfig.name].showPrefix
-        ? SOURCE_DETAILS[sourceConfig.name].showPrefix?.(source.payload)
-        : true
+export default function SourceFormContainer(props: SourceFormProps): JSX.Element {
+    return (
+        <Form logic={sourceWizardLogic} formKey="sourceConnectionDetails" enableFormOnSubmit>
+            <SourceFormComponent {...props} />
+        </Form>
+    )
+}
 
+export function SourceFormComponent({ sourceConfig, showPrefix = true, jobInputs }: SourceFormProps): JSX.Element {
     return (
-        <Form logic={sourceWizardLogic} formKey="sourceConnectionDetails" className="space-y-4" enableFormOnSubmit>
-            {showSourceFields && (
-                <Group name="payload">
-                    {SOURCE_DETAILS[sourceConfig.name].fields.map((field) => sourceFieldToElement(field, sourceConfig))}
-                </Group>
-            )}
+        <div className="space-y-4">
+            <Group name="payload">
+                {SOURCE_DETAILS[sourceConfig.name].fields.map((field) =>
+                    sourceFieldToElement(field, sourceConfig, jobInputs?.[field.name])
+                )}
+            </Group>
             {showPrefix && (
                 <LemonField name="prefix" label="Table Prefix (optional)">
                     <LemonInput className="ph-ignore-input" data-attr="prefix" placeholder="internal_" />
                 </LemonField>
             )}
-        </Form>
+        </div>
     )
 }

frontend/src/scenes/data-warehouse/settings/dataWarehouseSettingsLogic.ts

@@ -7,7 +7,7 @@ import posthog from 'posthog-js'
 import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'
 
 import { DatabaseSchemaDataWarehouseTable } from '~/queries/schema'
-import { DataWarehouseSettingsTab, ExternalDataSourceSchema, ExternalDataStripeSource } from '~/types'
+import { DataWarehouseSettingsTab, ExternalDataSource, ExternalDataSourceSchema } from '~/types'
 
 import type { dataWarehouseSettingsLogicType } from './dataWarehouseSettingsLogicType'
 
@@ -31,17 +31,17 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
         actions: [databaseTableListLogic, ['loadDatabase']],
     })),
     actions({
-        deleteSource: (source: ExternalDataStripeSource) => ({ source }),
-        reloadSource: (source: ExternalDataStripeSource) => ({ source }),
-        sourceLoadingFinished: (source: ExternalDataStripeSource) => ({ source }),
+        deleteSource: (source: ExternalDataSource) => ({ source }),
+        reloadSource: (source: ExternalDataSource) => ({ source }),
+        sourceLoadingFinished: (source: ExternalDataSource) => ({ source }),
         schemaLoadingFinished: (schema: ExternalDataSourceSchema) => ({ schema }),
         abortAnyRunningQuery: true,
         deleteSelfManagedTable: (tableId: string) => ({ tableId }),
         refreshSelfManagedTableSchema: (tableId: string) => ({ tableId }),
     }),
     loaders(({ cache, actions, values }) => ({
         dataWarehouseSources: [
-            null as PaginatedResponse<ExternalDataStripeSource> | null,
+            null as PaginatedResponse<ExternalDataSource> | null,
             {
                 loadSources: async (_, breakpoint) => {
                     await breakpoint(300)
@@ -59,7 +59,7 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
 
                     return res
                 },
-                updateSource: async (source: ExternalDataStripeSource) => {
+                updateSource: async (source: ExternalDataSource) => {
                     const updatedSource = await api.externalDataSources.update(source.id, source)
                     return {
                         ...values.dataWarehouseSources,
@@ -77,7 +77,7 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
                     // Optimistic UI updates before sending updates to the backend
                     const clonedSources = JSON.parse(
                         JSON.stringify(values.dataWarehouseSources?.results ?? [])
-                    ) as ExternalDataStripeSource[]
+                    ) as ExternalDataSource[]
                     const sourceIndex = clonedSources.findIndex((n) => n.schemas.find((m) => m.id === schema.id))
                     const schemaIndex = clonedSources[sourceIndex].schemas.findIndex((n) => n.id === schema.id)
                     clonedSources[sourceIndex].schemas[schemaIndex] = schema
@@ -166,7 +166,7 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
             // Optimistic UI updates before sending updates to the backend
             const clonedSources = JSON.parse(
                 JSON.stringify(values.dataWarehouseSources?.results ?? [])
-            ) as ExternalDataStripeSource[]
+            ) as ExternalDataSource[]
             const sourceIndex = clonedSources.findIndex((n) => n.id === source.id)
             clonedSources[sourceIndex].status = 'Running'
             clonedSources[sourceIndex].schemas = clonedSources[sourceIndex].schemas.map((n) => {

frontend/src/scenes/data-warehouse/settings/source/SourceConfiguration.tsx

@@ -0,0 +1,58 @@
+import { LemonBanner, LemonButton, LemonSkeleton } from '@posthog/lemon-ui'
+import { BindLogic, useValues } from 'kea'
+import { Form } from 'kea-forms'
+import { SourceFormComponent, SourceFormProps } from 'scenes/data-warehouse/external/forms/SourceForm'
+
+import { dataWarehouseSourceSettingsLogic } from './dataWarehouseSourceSettingsLogic'
+
+interface SourceConfigurationProps {
+    id: string
+}
+
+export const SourceConfiguration = ({ id }: SourceConfigurationProps): JSX.Element => {
+    const { sourceFieldConfig } = useValues(dataWarehouseSourceSettingsLogic({ id }))
+    return (
+        <BindLogic logic={dataWarehouseSourceSettingsLogic} props={{ id }}>
+            {sourceFieldConfig ? (
+                <UpdateSourceConnectionFormContainer id={id} sourceConfig={sourceFieldConfig} showPrefix={false} />
+            ) : (
+                <LemonSkeleton />
+            )}
+        </BindLogic>
+    )
+}
+
+interface UpdateSourceConnectionFormContainerProps extends SourceFormProps {
+    id: string
+}
+
+function UpdateSourceConnectionFormContainer(props: UpdateSourceConnectionFormContainerProps): JSX.Element {
+    const { source, sourceLoading } = useValues(dataWarehouseSourceSettingsLogic({ id: props.id }))
+
+    if (source?.source_type !== 'MSSQL' && source?.source_type !== 'MySQL' && source?.source_type !== 'Postgres') {
+        return (
+            <LemonBanner type="warning" className="mt-2">
+                <p>
+                    Only Postgres, MSSQL, and MySQL are configurable. Please delete and recreate your source if you need
+                    to connect to a new source of the same type.
+                </p>
+            </LemonBanner>
+        )
+    }
+    return (
+        <Form logic={dataWarehouseSourceSettingsLogic} formKey="sourceConfig" enableFormOnSubmit>
+            <SourceFormComponent {...props} jobInputs={source?.job_inputs} />
+            <div className="mt-4 flex flex-row justify-end gap-2">
+                <LemonButton
+                    loading={sourceLoading && !source}
+                    type="primary"
+                    center
+                    htmlType="submit"
+                    data-attr="source-update"
+                >
+                    Save
+                </LemonButton>
+            </div>
+        </Form>
+    )
+}