feat(data-exploration): convert funnel correlation to data exploration

frontend/src/queries/nodes/InsightViz/InsightContainer.tsx

@@ -30,12 +30,12 @@ import {
 import { PathCanvasLabel } from 'scenes/paths/PathsLabel'
 import { InsightLegend } from 'lib/components/InsightLegend/InsightLegend'
 import { InsightLegendButtonDataExploration } from 'lib/components/InsightLegend/InsightLegendButton'
-// import { FunnelCorrelation } from './views/Funnels/FunnelCorrelation'
 // import { AlertMessage } from 'lib/lemon-ui/AlertMessage'
 import { ComputationTimeWithRefresh } from './ComputationTimeWithRefresh'
 import { FunnelInsightDataExploration } from 'scenes/insights/views/Funnels/FunnelInsight'
 import { FunnelStepsTableDataExploration } from 'scenes/insights/views/Funnels/FunnelStepsTable'
 import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic'
+import { FunnelCorrelation } from 'scenes/insights/views/Funnels/FunnelCorrelation'
 
 const VIEW_MAP = {
     [`${InsightType.TRENDS}`]: <TrendInsight view={InsightType.TRENDS} />,
@@ -49,7 +49,7 @@ const VIEW_MAP = {
 export function InsightContainer({
     disableHeader,
     disableTable,
-    // disableCorrelationTable,
+    disableCorrelationTable,
     disableLastComputation,
     insightMode,
     context,
@@ -69,9 +69,6 @@ export function InsightContainer({
 
     const { activeView } = useValues(insightNavLogic(insightProps))
 
-    // const {
-    //     // correlationAnalysisAvailable
-    // } = useValues(funnelLogic(insightProps))
     const { isFunnelWithEnoughSteps, hasFunnelResults, areExclusionFiltersValid } = useValues(
         funnelDataLogic(insightProps)
     )
@@ -243,9 +240,7 @@ export function InsightContainer({
                 </div>
             </Card>
             {renderTable()}
-            {/* {!disableCorrelationTable && correlationAnalysisAvailable && activeView === InsightType.FUNNELS && (
-                <FunnelCorrelation />
-            )} */}
+            {!disableCorrelationTable && activeView === InsightType.FUNNELS && <FunnelCorrelation />}
         </>
     )
 }

frontend/src/scenes/funnels/funnelCorrelationDetailsLogic.test.ts

@@ -0,0 +1,85 @@
+import { expectLogic } from 'kea-test-utils'
+import { initKeaTests } from '~/test/init'
+import { FunnelCorrelationResultsType, FunnelCorrelationType, InsightLogicProps, InsightType } from '~/types'
+
+import { funnelCorrelationDetailsLogic } from './funnelCorrelationDetailsLogic'
+
+const funnelResults = [
+    {
+        action_id: '$pageview',
+        count: 19,
+        name: '$pageview',
+        order: 0,
+        type: 'events',
+    },
+    {
+        action_id: '$pageview',
+        count: 7,
+        name: '$pageview',
+        order: 1,
+        type: 'events',
+    },
+    {
+        action_id: '$pageview',
+        count: 4,
+        name: '$pageview',
+        order: 2,
+        type: 'events',
+    },
+]
+
+describe('funnelCorrelationDetailsLogic', () => {
+    let logic: ReturnType<typeof funnelCorrelationDetailsLogic.build>
+
+    beforeEach(() => {
+        initKeaTests(false)
+    })
+
+    const defaultProps: InsightLogicProps = {
+        dashboardItemId: undefined,
+        cachedInsight: {
+            short_id: undefined,
+            filters: {
+                insight: InsightType.FUNNELS,
+                actions: [
+                    { id: '$pageview', order: 0 },
+                    { id: '$pageview', order: 1 },
+                ],
+            },
+            result: funnelResults,
+        },
+    }
+
+    beforeEach(async () => {
+        logic = funnelCorrelationDetailsLogic(defaultProps)
+        logic.mount()
+    })
+
+    describe('correlationMatrixAndScore', () => {
+        it('returns calculated values based on selected details', async () => {
+            await expectLogic(logic, () =>
+                logic.actions.setFunnelCorrelationDetails({
+                    event: { event: 'some event', elements: [], properties: {} },
+                    success_people_url: '',
+                    failure_people_url: '',
+                    success_count: 2,
+                    failure_count: 4,
+                    odds_ratio: 3,
+                    correlation_type: FunnelCorrelationType.Success,
+                    result_type: FunnelCorrelationResultsType.Events,
+                })
+            ).toMatchValues({
+                correlationMatrixAndScore: {
+                    correlationScore: expect.anything(),
+                    correlationScoreStrength: 'weak',
+                    truePositive: 2,
+                    falsePositive: 2,
+                    trueNegative: 11,
+                    falseNegative: 4,
+                },
+            })
+
+            expect(logic.values.correlationMatrixAndScore.correlationScore).toBeCloseTo(0.204)
+        })
+    })
+})

frontend/src/scenes/funnels/funnelCorrelationDetailsLogic.ts

@@ -0,0 +1,105 @@
+import { kea, props, key, path, connect, selectors, reducers, actions } from 'kea'
+import { keyForInsightLogicProps } from 'scenes/insights/sharedUtils'
+import { FunnelCorrelation, InsightLogicProps } from '~/types'
+
+import { insightLogic } from 'scenes/insights/insightLogic'
+import { funnelLogic } from './funnelLogic'
+import { funnelDataLogic } from './funnelDataLogic'
+
+import type { funnelCorrelationDetailsLogicType } from './funnelCorrelationDetailsLogicType'
+
+export const funnelCorrelationDetailsLogic = kea<funnelCorrelationDetailsLogicType>([
+    props({} as InsightLogicProps),
+    key(keyForInsightLogicProps('insight_funnel')),
+    path((key) => ['scenes', 'funnels', 'funnelCorrelationDetailsLogic', key]),
+    connect((props: InsightLogicProps) => ({
+        values: [
+            insightLogic(props),
+            ['isUsingDataExploration'],
+            funnelLogic(props),
+            ['steps as legacySteps'],
+            funnelDataLogic(props),
+            ['steps as dataExplorationSteps'],
+        ],
+    })),
+
+    actions({
+        setFunnelCorrelationDetails: (payload: FunnelCorrelation | null) => ({ payload }),
+    }),
+
+    reducers({
+        funnelCorrelationDetails: [
+            null as null | FunnelCorrelation,
+            {
+                setFunnelCorrelationDetails: (_, { payload }) => payload,
+            },
+        ],
+    }),
+
+    selectors({
+        steps: [
+            (s) => [s.isUsingDataExploration, s.dataExplorationSteps, s.legacySteps],
+            (isUsingDataExploration, dataExplorationApiParams, legacyApiParams) => {
+                return isUsingDataExploration ? dataExplorationApiParams : legacyApiParams
+            },
+        ],
+
+        correlationMatrixAndScore: [
+            (s) => [s.funnelCorrelationDetails, s.steps],
+            (
+                funnelCorrelationDetails,
+                steps
+            ): {
+                truePositive: number
+                falsePositive: number
+                trueNegative: number
+                falseNegative: number
+                correlationScore: number
+                correlationScoreStrength: 'weak' | 'moderate' | 'strong' | null
+            } => {
+                if (!funnelCorrelationDetails) {
+                    return {
+                        truePositive: 0,
+                        falsePositive: 0,
+                        trueNegative: 0,
+                        falseNegative: 0,
+                        correlationScore: 0,
+                        correlationScoreStrength: null,
+                    }
+                }
+
+                const successTotal = steps[steps.length - 1].count
+                const failureTotal = steps[0].count - successTotal
+                const success = funnelCorrelationDetails.success_count
+                const failure = funnelCorrelationDetails.failure_count
+
+                const truePositive = success // has property, converted
+                const falseNegative = failure // has property, but dropped off
+                const trueNegative = failureTotal - failure // doesn't have property, dropped off
+                const falsePositive = successTotal - success // doesn't have property, converted
+
+                // Phi coefficient: https://en.wikipedia.org/wiki/Phi_coefficient
+                const correlationScore =
+                    (truePositive * trueNegative - falsePositive * falseNegative) /
+                    Math.sqrt(
+                        (truePositive + falsePositive) *
+                            (truePositive + falseNegative) *
+                            (trueNegative + falsePositive) *
+                            (trueNegative + falseNegative)
+                    )
+
+                const correlationScoreStrength =
+                    Math.abs(correlationScore) > 0.5 ? 'strong' : Math.abs(correlationScore) > 0.3 ? 'moderate' : 'weak'
+
+                return {
+                    correlationScore,
+                    truePositive,
+                    falsePositive,
+                    trueNegative,
+                    falseNegative,
+                    correlationScoreStrength,
+                }
+            },
+        ],
+    }),
+])

frontend/src/scenes/funnels/funnelCorrelationFeedbackLogic.test.ts

@@ -0,0 +1,109 @@
+import posthog from 'posthog-js'
+import { expectLogic } from 'kea-test-utils'
+import { initKeaTests } from '~/test/init'
+import { eventUsageLogic } from 'lib/utils/eventUsageLogic'
+import { AvailableFeature, InsightLogicProps, InsightType } from '~/types'
+import { useAvailableFeatures } from '~/mocks/features'
+import { funnelCorrelationFeedbackLogic } from './funnelCorrelationFeedbackLogic'
+
+describe('funnelCorrelationFeedbackLogic', () => {
+    let logic: ReturnType<typeof funnelCorrelationFeedbackLogic.build>
+
+    beforeEach(() => {
+        useAvailableFeatures([AvailableFeature.CORRELATION_ANALYSIS])
+        initKeaTests(false)
+    })
+
+    const defaultProps: InsightLogicProps = {
+        dashboardItemId: undefined,
+        cachedInsight: {
+            short_id: undefined,
+            filters: {
+                insight: InsightType.FUNNELS,
+                actions: [
+                    { id: '$pageview', order: 0 },
+                    { id: '$pageview', order: 1 },
+                ],
+            },
+            result: [],
+        },
+    }
+
+    beforeEach(async () => {
+        logic = funnelCorrelationFeedbackLogic(defaultProps)
+        logic.mount()
+    })
+
+    it('opens detailed feedback on selecting a valid rating', async () => {
+        await expectLogic(logic, () => {
+            logic.actions.setCorrelationFeedbackRating(1)
+        })
+            .toMatchValues(logic, {
+                correlationFeedbackRating: 1,
+            })
+            .toDispatchActions(logic, [
+                (action) =>
+                    action.type === logic.actionTypes.setCorrelationDetailedFeedbackVisible &&
+                    action.payload.visible === true,
+            ])
+            .toMatchValues(logic, {
+                correlationDetailedFeedbackVisible: true,
+            })
+    })
+
+    it('doesnt opens detailed feedback on selecting an invalid rating', async () => {
+        await expectLogic(logic, () => {
+            logic.actions.setCorrelationFeedbackRating(0)
+        })
+            .toMatchValues(logic, {
+                correlationFeedbackRating: 0,
+            })
+            .toDispatchActions(logic, [
+                (action) =>
+                    action.type === logic.actionTypes.setCorrelationDetailedFeedbackVisible &&
+                    action.payload.visible === false,
+            ])
+            .toMatchValues(logic, {
+                correlationDetailedFeedbackVisible: false,
+            })
+    })
+
+    it('captures emoji feedback properly', async () => {
+        jest.spyOn(posthog, 'capture')
+        await expectLogic(logic, () => {
+            logic.actions.setCorrelationFeedbackRating(1)
+        })
+            .toMatchValues(logic, {
+                // reset after sending feedback
+                correlationFeedbackRating: 1,
+            })
+            .toDispatchActions(eventUsageLogic, ['reportCorrelationAnalysisFeedback'])
+
+        expect(posthog.capture).toBeCalledWith('correlation analysis feedback', { rating: 1 })
+    })
+
+    it('goes away on sending feedback, capturing it properly', async () => {
+        jest.spyOn(posthog, 'capture')
+        await expectLogic(logic, () => {
+            logic.actions.setCorrelationFeedbackRating(2)
+            logic.actions.setCorrelationDetailedFeedback('tests')
+            logic.actions.sendCorrelationAnalysisFeedback()
+        })
+            .toMatchValues(logic, {
+                // reset after sending feedback
+                correlationFeedbackRating: 0,
+                correlationDetailedFeedback: '',
+                correlationFeedbackHidden: true,
+            })
+            .toDispatchActions(eventUsageLogic, ['reportCorrelationAnalysisDetailedFeedback'])
+            .toFinishListeners()
+
+        await expectLogic(eventUsageLogic).toFinishListeners()
+
+        expect(posthog.capture).toBeCalledWith('correlation analysis feedback', { rating: 2 })
+        expect(posthog.capture).toBeCalledWith('correlation analysis detailed feedback', {
+            rating: 2,
+            comments: 'tests',
+        })
+    })
+})