diff --git a/ee/clickhouse/queries/experiments/funnel_experiment_result.py b/ee/clickhouse/queries/experiments/funnel_experiment_result.py index 20f9631a4dc4f..e311657cc52c7 100644 --- a/ee/clickhouse/queries/experiments/funnel_experiment_result.py +++ b/ee/clickhouse/queries/experiments/funnel_experiment_result.py @@ -8,7 +8,7 @@ from posthog.constants import ExperimentNoResultsErrorKeys from posthog.hogql_queries.experiments import CONTROL_VARIANT_KEY -from posthog.hogql_queries.experiments.funnel_statistics import ( +from posthog.hogql_queries.experiments.funnels_statistics import ( are_results_significant, calculate_credible_intervals, calculate_probabilities, diff --git a/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py b/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py index 374720c3141f1..55fca255ed9ca 100644 --- a/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py +++ b/ee/clickhouse/queries/experiments/test_funnel_experiment_result.py @@ -4,13 +4,13 @@ from flaky import flaky -from posthog.hogql_queries.experiments.funnel_statistics import ( +from posthog.hogql_queries.experiments.funnels_statistics import ( are_results_significant, calculate_expected_loss, calculate_probabilities, calculate_credible_intervals as calculate_funnel_credible_intervals, ) -from posthog.schema import ExperimentSignificanceCode, ExperimentVariantFunnelResult +from posthog.schema import ExperimentSignificanceCode, ExperimentVariantFunnelsBaseStats Probability = float @@ -25,7 +25,7 @@ def logbeta(x: int, y: int) -> float: def calculate_probability_of_winning_for_target( - target_variant: ExperimentVariantFunnelResult, other_variants: list[ExperimentVariantFunnelResult] + target_variant: ExperimentVariantFunnelsBaseStats, other_variants: list[ExperimentVariantFunnelsBaseStats] ) -> Probability: """ Calculates the probability of winning for target variant. @@ -146,8 +146,8 @@ def probability_D_beats_A_B_and_C( @flaky(max_runs=10, min_passes=1) class TestFunnelExperimentCalculator(unittest.TestCase): def test_calculate_results(self): - variant_test = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10) - variant_control = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=18) + variant_test = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10) + variant_control = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=18) _, probability = calculate_probabilities(variant_control, [variant_test]) self.assertAlmostEqual(probability, 0.918, places=2) @@ -164,8 +164,8 @@ def test_calculate_results(self): self.assertAlmostEqual(credible_intervals[variant_test.key][1], 0.9494, places=3) def test_simulation_result_is_close_to_closed_form_solution(self): - variant_test = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10) - variant_control = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=18) + variant_test = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10) + variant_control = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=18) _, probability = calculate_probabilities(variant_control, [variant_test]) self.assertAlmostEqual(probability, 0.918, places=1) @@ -174,9 +174,9 @@ def test_simulation_result_is_close_to_closed_form_solution(self): self.assertAlmostEqual(probability, alternative_probability, places=1) def test_calculate_results_for_two_test_variants(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10) - variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=3) - variant_control = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=18) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=3) + variant_control = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=18) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(sum(probabilities), 1) @@ -210,9 +210,9 @@ def test_calculate_results_for_two_test_variants(self): self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.9894, places=3) def test_calculate_results_for_two_test_variants_almost_equal(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=120, failure_count=60) - variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=110, failure_count=52) - variant_control = ExperimentVariantFunnelResult(key="C", success_count=130, failure_count=65) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=120, failure_count=60) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=110, failure_count=52) + variant_control = ExperimentVariantFunnelsBaseStats(key="C", success_count=130, failure_count=65) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(sum(probabilities), 1) @@ -245,8 +245,8 @@ def test_calculate_results_for_two_test_variants_almost_equal(self): self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.7460, places=3) def test_absolute_loss_less_than_one_percent_but_not_significant(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=286, failure_count=2014) - variant_control = ExperimentVariantFunnelResult(key="B", success_count=267, failure_count=2031) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=286, failure_count=2014) + variant_control = ExperimentVariantFunnelsBaseStats(key="B", success_count=267, failure_count=2031) probabilities = calculate_probabilities(variant_control, [variant_test_1]) self.assertAlmostEqual(sum(probabilities), 1) @@ -267,10 +267,10 @@ def test_absolute_loss_less_than_one_percent_but_not_significant(self): self.assertAlmostEqual(credible_intervals[variant_test_1.key][1], 0.1384, places=3) def test_calculate_results_for_three_test_variants(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=10) - variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=3) - variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=30) - variant_control = ExperimentVariantFunnelResult(key="D", success_count=100, failure_count=18) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=10) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=3) + variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=30) + variant_control = ExperimentVariantFunnelsBaseStats(key="D", success_count=100, failure_count=18) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2, variant_test_3]) self.assertAlmostEqual(sum(probabilities), 1) @@ -313,10 +313,10 @@ def test_calculate_results_for_three_test_variants(self): self.assertAlmostEqual(credible_intervals[variant_test_3.key][1], 0.8332, places=3) def test_calculate_results_for_three_test_variants_almost_equal(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=120, failure_count=60) - variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=110, failure_count=52) - variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=46) - variant_control = ExperimentVariantFunnelResult(key="D", success_count=130, failure_count=65) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=120, failure_count=60) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=110, failure_count=52) + variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=46) + variant_control = ExperimentVariantFunnelsBaseStats(key="D", success_count=130, failure_count=65) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2, variant_test_3]) self.assertAlmostEqual(sum(probabilities), 1) @@ -357,10 +357,10 @@ def test_calculate_results_for_three_test_variants_almost_equal(self): self.assertAlmostEqual(credible_intervals[variant_test_3.key][1], 0.7547, places=3) def test_calculate_results_for_three_test_variants_much_better_than_control(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=130, failure_count=60) - variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=135, failure_count=62) - variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=132, failure_count=60) - variant_control = ExperimentVariantFunnelResult(key="D", success_count=80, failure_count=65) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=130, failure_count=60) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=135, failure_count=62) + variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=132, failure_count=60) + variant_control = ExperimentVariantFunnelsBaseStats(key="D", success_count=80, failure_count=65) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2, variant_test_3]) self.assertAlmostEqual(sum(probabilities), 1) @@ -392,14 +392,14 @@ def test_calculate_results_for_three_test_variants_much_better_than_control(self self.assertAlmostEqual(credible_intervals[variant_test_3.key][1], 0.7488, places=3) def test_calculate_results_for_seven_test_variants(self): - variant_test_1 = ExperimentVariantFunnelResult(key="A", success_count=100, failure_count=17) - variant_test_2 = ExperimentVariantFunnelResult(key="B", success_count=100, failure_count=16) - variant_test_3 = ExperimentVariantFunnelResult(key="C", success_count=100, failure_count=30) - variant_test_4 = ExperimentVariantFunnelResult(key="D", success_count=100, failure_count=31) - variant_test_5 = ExperimentVariantFunnelResult(key="E", success_count=100, failure_count=29) - variant_test_6 = ExperimentVariantFunnelResult(key="F", success_count=100, failure_count=32) - variant_test_7 = ExperimentVariantFunnelResult(key="G", success_count=100, failure_count=33) - variant_control = ExperimentVariantFunnelResult(key="H", success_count=100, failure_count=18) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="A", success_count=100, failure_count=17) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="B", success_count=100, failure_count=16) + variant_test_3 = ExperimentVariantFunnelsBaseStats(key="C", success_count=100, failure_count=30) + variant_test_4 = ExperimentVariantFunnelsBaseStats(key="D", success_count=100, failure_count=31) + variant_test_5 = ExperimentVariantFunnelsBaseStats(key="E", success_count=100, failure_count=29) + variant_test_6 = ExperimentVariantFunnelsBaseStats(key="F", success_count=100, failure_count=32) + variant_test_7 = ExperimentVariantFunnelsBaseStats(key="G", success_count=100, failure_count=33) + variant_control = ExperimentVariantFunnelsBaseStats(key="H", success_count=100, failure_count=18) probabilities = calculate_probabilities( variant_control, @@ -487,8 +487,8 @@ def test_calculate_results_for_seven_test_variants(self): self.assertAlmostEqual(credible_intervals[variant_test_7.key][1], 0.8174, places=3) def test_calculate_results_control_is_significant(self): - variant_test = ExperimentVariantFunnelResult(key="test", success_count=100, failure_count=18) - variant_control = ExperimentVariantFunnelResult(key="control", success_count=100, failure_count=10) + variant_test = ExperimentVariantFunnelsBaseStats(key="test", success_count=100, failure_count=18) + variant_control = ExperimentVariantFunnelsBaseStats(key="control", success_count=100, failure_count=10) probabilities = calculate_probabilities(variant_control, [variant_test]) @@ -507,13 +507,13 @@ def test_calculate_results_control_is_significant(self): self.assertAlmostEqual(credible_intervals[variant_test.key][1], 0.9010, places=3) def test_calculate_results_many_variants_control_is_significant(self): - variant_test_1 = ExperimentVariantFunnelResult(key="test_1", success_count=100, failure_count=20) - variant_test_2 = ExperimentVariantFunnelResult(key="test_2", success_count=100, failure_count=21) - variant_test_3 = ExperimentVariantFunnelResult(key="test_3", success_count=100, failure_count=22) - variant_test_4 = ExperimentVariantFunnelResult(key="test_4", success_count=100, failure_count=23) - variant_test_5 = ExperimentVariantFunnelResult(key="test_5", success_count=100, failure_count=24) - variant_test_6 = ExperimentVariantFunnelResult(key="test_6", success_count=100, failure_count=25) - variant_control = ExperimentVariantFunnelResult(key="control", success_count=100, failure_count=10) + variant_test_1 = ExperimentVariantFunnelsBaseStats(key="test_1", success_count=100, failure_count=20) + variant_test_2 = ExperimentVariantFunnelsBaseStats(key="test_2", success_count=100, failure_count=21) + variant_test_3 = ExperimentVariantFunnelsBaseStats(key="test_3", success_count=100, failure_count=22) + variant_test_4 = ExperimentVariantFunnelsBaseStats(key="test_4", success_count=100, failure_count=23) + variant_test_5 = ExperimentVariantFunnelsBaseStats(key="test_5", success_count=100, failure_count=24) + variant_test_6 = ExperimentVariantFunnelsBaseStats(key="test_6", success_count=100, failure_count=25) + variant_control = ExperimentVariantFunnelsBaseStats(key="control", success_count=100, failure_count=10) variants_test = [ variant_test_1, diff --git a/ee/clickhouse/queries/experiments/test_trend_experiment_result.py b/ee/clickhouse/queries/experiments/test_trend_experiment_result.py index 4799e3026d624..de983e6f1496c 100644 --- a/ee/clickhouse/queries/experiments/test_trend_experiment_result.py +++ b/ee/clickhouse/queries/experiments/test_trend_experiment_result.py @@ -4,13 +4,13 @@ from flaky import flaky -from posthog.hogql_queries.experiments.trend_statistics import ( +from posthog.hogql_queries.experiments.trends_statistics import ( are_results_significant, calculate_credible_intervals, calculate_p_value, calculate_probabilities, ) -from posthog.schema import ExperimentSignificanceCode, ExperimentVariantTrendBaseStats +from posthog.schema import ExperimentSignificanceCode, ExperimentVariantTrendsBaseStats Probability = float @@ -23,7 +23,7 @@ def logbeta(x: float, y: float) -> float: # Helper function to calculate probability using a different method than the one used in actual code # calculation: https://www.evanmiller.org/bayesian-ab-testing.html#count_ab def calculate_probability_of_winning_for_target_count_data( - target_variant: ExperimentVariantTrendBaseStats, other_variants: list[ExperimentVariantTrendBaseStats] + target_variant: ExperimentVariantTrendsBaseStats, other_variants: list[ExperimentVariantTrendsBaseStats] ) -> Probability: """ Calculates the probability of winning for target variant. @@ -97,8 +97,8 @@ def probability_C_beats_A_and_B_count_data( @flaky(max_runs=10, min_passes=1) class TestTrendExperimentCalculator(unittest.TestCase): def test_calculate_results(self): - variant_control = ExperimentVariantTrendBaseStats(key="A", count=20, exposure=1, absolute_exposure=200) - variant_test = ExperimentVariantTrendBaseStats(key="B", count=30, exposure=1, absolute_exposure=200) + variant_control = ExperimentVariantTrendsBaseStats(key="A", count=20, exposure=1, absolute_exposure=200) + variant_test = ExperimentVariantTrendsBaseStats(key="B", count=30, exposure=1, absolute_exposure=200) probabilities = calculate_probabilities(variant_control, [variant_test]) self.assertAlmostEqual(probabilities[1], 0.92, places=1) @@ -117,8 +117,8 @@ def test_calculate_results(self): self.assertAlmostEqual(credible_intervals[variant_test.key][1], 0.2141, places=3) def test_calculate_results_small_numbers(self): - variant_control = ExperimentVariantTrendBaseStats(key="A", count=2, exposure=1, absolute_exposure=200) - variant_test = ExperimentVariantTrendBaseStats(key="B", count=1, exposure=1, absolute_exposure=200) + variant_control = ExperimentVariantTrendsBaseStats(key="A", count=2, exposure=1, absolute_exposure=200) + variant_test = ExperimentVariantTrendsBaseStats(key="B", count=1, exposure=1, absolute_exposure=200) probabilities = calculate_probabilities(variant_control, [variant_test]) self.assertAlmostEqual(probabilities[1], 0.31, places=1) @@ -145,9 +145,9 @@ def test_calculate_count_data_probability(self): self.assertAlmostEqual(probability, probability2) def test_calculate_results_with_three_variants(self): - variant_control = ExperimentVariantTrendBaseStats(key="A", count=20, exposure=1, absolute_exposure=200) - variant_test_1 = ExperimentVariantTrendBaseStats(key="B", count=26, exposure=1, absolute_exposure=200) - variant_test_2 = ExperimentVariantTrendBaseStats(key="C", count=19, exposure=1, absolute_exposure=200) + variant_control = ExperimentVariantTrendsBaseStats(key="A", count=20, exposure=1, absolute_exposure=200) + variant_test_1 = ExperimentVariantTrendsBaseStats(key="B", count=26, exposure=1, absolute_exposure=200) + variant_test_2 = ExperimentVariantTrendsBaseStats(key="C", count=19, exposure=1, absolute_exposure=200) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(probabilities[0], 0.16, places=1) @@ -171,9 +171,9 @@ def test_calculate_results_with_three_variants(self): self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.1484, places=3) def test_calculate_significance_when_target_variants_underperform(self): - variant_control = ExperimentVariantTrendBaseStats(key="A", count=250, exposure=1, absolute_exposure=200) - variant_test_1 = ExperimentVariantTrendBaseStats(key="B", count=180, exposure=1, absolute_exposure=200) - variant_test_2 = ExperimentVariantTrendBaseStats(key="C", count=50, exposure=1, absolute_exposure=200) + variant_control = ExperimentVariantTrendsBaseStats(key="A", count=250, exposure=1, absolute_exposure=200) + variant_test_1 = ExperimentVariantTrendsBaseStats(key="B", count=180, exposure=1, absolute_exposure=200) + variant_test_2 = ExperimentVariantTrendsBaseStats(key="C", count=50, exposure=1, absolute_exposure=200) # in this case, should choose B as best test variant p_value = calculate_p_value(variant_control, [variant_test_1, variant_test_2]) @@ -187,7 +187,7 @@ def test_calculate_significance_when_target_variants_underperform(self): self.assertEqual(significant, ExperimentSignificanceCode.LOW_WIN_PROBABILITY) # new B variant is worse, such that control probability ought to be high enough - variant_test_1 = ExperimentVariantTrendBaseStats(key="B", count=100, exposure=1, absolute_exposure=200) + variant_test_1 = ExperimentVariantTrendsBaseStats(key="B", count=100, exposure=1, absolute_exposure=200) significant, p_value = are_results_significant( variant_control, [variant_test_1, variant_test_2], [0.95, 0.03, 0.02] @@ -204,9 +204,9 @@ def test_calculate_significance_when_target_variants_underperform(self): self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.3295, places=3) def test_results_with_different_exposures(self): - variant_control = ExperimentVariantTrendBaseStats(key="A", count=50, exposure=1.3, absolute_exposure=260) - variant_test_1 = ExperimentVariantTrendBaseStats(key="B", count=30, exposure=1.8, absolute_exposure=360) - variant_test_2 = ExperimentVariantTrendBaseStats(key="C", count=20, exposure=0.7, absolute_exposure=140) + variant_control = ExperimentVariantTrendsBaseStats(key="A", count=50, exposure=1.3, absolute_exposure=260) + variant_test_1 = ExperimentVariantTrendsBaseStats(key="B", count=30, exposure=1.8, absolute_exposure=360) + variant_test_2 = ExperimentVariantTrendsBaseStats(key="C", count=20, exposure=0.7, absolute_exposure=140) probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2]) # a is control self.assertAlmostEqual(probabilities[0], 0.86, places=1) diff --git a/ee/clickhouse/queries/experiments/trend_experiment_result.py b/ee/clickhouse/queries/experiments/trend_experiment_result.py index 89aae12d05895..0971120f2366a 100644 --- a/ee/clickhouse/queries/experiments/trend_experiment_result.py +++ b/ee/clickhouse/queries/experiments/trend_experiment_result.py @@ -17,7 +17,7 @@ UNIQUE_USERS, ExperimentNoResultsErrorKeys, ) -from posthog.hogql_queries.experiments.trend_statistics import ( +from posthog.hogql_queries.experiments.trends_statistics import ( are_results_significant, calculate_credible_intervals, calculate_probabilities, diff --git a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png index b54d4facb0bfe..e3e93b3dd9678 100644 Binary files a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png and b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png differ diff --git a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png index 332ea24ce3084..d4377362d9270 100644 Binary files a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png and b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png differ diff --git a/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--dark.png b/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--dark.png index c2c9bf4b33166..8ac461d15d0bb 100644 Binary files a/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--dark.png and b/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--dark.png differ diff --git a/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--light.png b/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--light.png index 74ac1c339682a..bbb59b784ba62 100644 Binary files a/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--light.png and b/frontend/__snapshots__/exporter-exporter--trends-line-insight-legend--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--dark.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--dark.png index e86df67429be0..ebceafa1ecce5 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--dark.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-edit--dark.png differ diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 3e3645da4a059..a9dc7be1bf476 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -477,10 +477,10 @@ "$ref": "#/definitions/ErrorTrackingQuery" }, { - "$ref": "#/definitions/ExperimentFunnelQuery" + "$ref": "#/definitions/ExperimentFunnelsQuery" }, { - "$ref": "#/definitions/ExperimentTrendQuery" + "$ref": "#/definitions/ExperimentTrendsQuery" } ] }, @@ -1250,7 +1250,7 @@ ], "type": "object" }, - "CachedExperimentFunnelQueryResponse": { + "CachedExperimentFunnelsQueryResponse": { "additionalProperties": false, "properties": { "cache_key": { @@ -1264,9 +1264,22 @@ "description": "What triggered the calculation of the query, leave empty if user/immediate", "type": "string" }, + "credible_intervals": { + "additionalProperties": { + "items": { + "type": "number" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "type": "object" + }, + "expected_loss": { + "type": "number" + }, "insight": { - "const": "FUNNELS", - "type": "string" + "$ref": "#/definitions/FunnelsQueryResponse" }, "is_cached": { "type": "boolean" @@ -1279,32 +1292,49 @@ "format": "date-time", "type": "string" }, + "probability": { + "additionalProperties": { + "type": "number" + }, + "type": "object" + }, "query_status": { "$ref": "#/definitions/QueryStatus", "description": "Query status indicates whether next to the provided data, a query is still running." }, - "results": { - "additionalProperties": { - "$ref": "#/definitions/ExperimentVariantFunnelResult" - }, - "type": "object" + "significance_code": { + "$ref": "#/definitions/ExperimentSignificanceCode" + }, + "significant": { + "type": "boolean" }, "timezone": { "type": "string" + }, + "variants": { + "items": { + "$ref": "#/definitions/ExperimentVariantFunnelsBaseStats" + }, + "type": "array" } }, "required": [ "cache_key", + "credible_intervals", + "expected_loss", "insight", "is_cached", "last_refresh", "next_allowed_client_refresh", - "results", - "timezone" + "probability", + "significance_code", + "significant", + "timezone", + "variants" ], "type": "object" }, - "CachedExperimentTrendQueryResponse": { + "CachedExperimentTrendsQueryResponse": { "additionalProperties": false, "properties": { "cache_key": { @@ -1367,7 +1397,7 @@ }, "variants": { "items": { - "$ref": "#/definitions/ExperimentVariantTrendBaseStats" + "$ref": "#/definitions/ExperimentVariantTrendsBaseStats" }, "type": "array" } @@ -3651,18 +3681,51 @@ { "additionalProperties": false, "properties": { + "credible_intervals": { + "additionalProperties": { + "items": { + "type": "number" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "type": "object" + }, + "expected_loss": { + "type": "number" + }, "insight": { - "const": "FUNNELS", - "type": "string" + "$ref": "#/definitions/FunnelsQueryResponse" }, - "results": { + "probability": { "additionalProperties": { - "$ref": "#/definitions/ExperimentVariantFunnelResult" + "type": "number" }, "type": "object" + }, + "significance_code": { + "$ref": "#/definitions/ExperimentSignificanceCode" + }, + "significant": { + "type": "boolean" + }, + "variants": { + "items": { + "$ref": "#/definitions/ExperimentVariantFunnelsBaseStats" + }, + "type": "array" } }, - "required": ["insight", "results"], + "required": [ + "credible_intervals", + "expected_loss", + "insight", + "probability", + "significance_code", + "significant", + "variants" + ], "type": "object" }, { @@ -3699,7 +3762,7 @@ }, "variants": { "items": { - "$ref": "#/definitions/ExperimentVariantTrendBaseStats" + "$ref": "#/definitions/ExperimentVariantTrendsBaseStats" }, "type": "array" } @@ -3830,10 +3893,10 @@ "$ref": "#/definitions/ErrorTrackingQuery" }, { - "$ref": "#/definitions/ExperimentFunnelQuery" + "$ref": "#/definitions/ExperimentFunnelsQuery" }, { - "$ref": "#/definitions/ExperimentTrendQuery" + "$ref": "#/definitions/ExperimentTrendsQuery" } ], "description": "Source of the events" @@ -5065,14 +5128,14 @@ "required": ["columns", "hogql", "results", "types"], "type": "object" }, - "ExperimentFunnelQuery": { + "ExperimentFunnelsQuery": { "additionalProperties": false, "properties": { "experiment_id": { "type": "integer" }, "kind": { - "const": "ExperimentFunnelQuery", + "const": "ExperimentFunnelsQuery", "type": "string" }, "modifiers": { @@ -5080,7 +5143,7 @@ "description": "Modifiers used when performing the query" }, "response": { - "$ref": "#/definitions/ExperimentFunnelQueryResponse" + "$ref": "#/definitions/ExperimentFunnelsQueryResponse" }, "source": { "$ref": "#/definitions/FunnelsQuery" @@ -5089,28 +5152,61 @@ "required": ["experiment_id", "kind", "source"], "type": "object" }, - "ExperimentFunnelQueryResponse": { + "ExperimentFunnelsQueryResponse": { "additionalProperties": false, "properties": { + "credible_intervals": { + "additionalProperties": { + "items": { + "type": "number" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "type": "object" + }, + "expected_loss": { + "type": "number" + }, "insight": { - "const": "FUNNELS", - "type": "string" + "$ref": "#/definitions/FunnelsQueryResponse" }, - "results": { + "probability": { "additionalProperties": { - "$ref": "#/definitions/ExperimentVariantFunnelResult" + "type": "number" }, "type": "object" + }, + "significance_code": { + "$ref": "#/definitions/ExperimentSignificanceCode" + }, + "significant": { + "type": "boolean" + }, + "variants": { + "items": { + "$ref": "#/definitions/ExperimentVariantFunnelsBaseStats" + }, + "type": "array" } }, - "required": ["insight", "results"], + "required": [ + "insight", + "variants", + "probability", + "significant", + "significance_code", + "expected_loss", + "credible_intervals" + ], "type": "object" }, "ExperimentSignificanceCode": { "enum": ["significant", "not_enough_exposure", "low_win_probability", "high_loss", "high_p_value"], "type": "string" }, - "ExperimentTrendQuery": { + "ExperimentTrendsQuery": { "additionalProperties": false, "properties": { "count_query": { @@ -5123,7 +5219,7 @@ "$ref": "#/definitions/TrendsQuery" }, "kind": { - "const": "ExperimentTrendQuery", + "const": "ExperimentTrendsQuery", "type": "string" }, "modifiers": { @@ -5131,13 +5227,13 @@ "description": "Modifiers used when performing the query" }, "response": { - "$ref": "#/definitions/ExperimentTrendQueryResponse" + "$ref": "#/definitions/ExperimentTrendsQueryResponse" } }, "required": ["count_query", "experiment_id", "kind"], "type": "object" }, - "ExperimentTrendQueryResponse": { + "ExperimentTrendsQueryResponse": { "additionalProperties": false, "properties": { "credible_intervals": { @@ -5171,7 +5267,7 @@ }, "variants": { "items": { - "$ref": "#/definitions/ExperimentVariantTrendBaseStats" + "$ref": "#/definitions/ExperimentVariantTrendsBaseStats" }, "type": "array" } @@ -5187,7 +5283,7 @@ ], "type": "object" }, - "ExperimentVariantFunnelResult": { + "ExperimentVariantFunnelsBaseStats": { "additionalProperties": false, "properties": { "failure_count": { @@ -5203,7 +5299,7 @@ "required": ["key", "success_count", "failure_count"], "type": "object" }, - "ExperimentVariantTrendBaseStats": { + "ExperimentVariantTrendsBaseStats": { "additionalProperties": false, "properties": { "absolute_exposure": { @@ -7519,8 +7615,8 @@ "WebStatsTableQuery", "WebExternalClicksTableQuery", "WebGoalsQuery", - "ExperimentFunnelQuery", - "ExperimentTrendQuery", + "ExperimentFunnelsQuery", + "ExperimentTrendsQuery", "DatabaseSchemaQuery", "SuggestedQuestionsQuery", "TeamTaxonomyQuery", @@ -8786,18 +8882,51 @@ { "additionalProperties": false, "properties": { + "credible_intervals": { + "additionalProperties": { + "items": { + "type": "number" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "type": "object" + }, + "expected_loss": { + "type": "number" + }, "insight": { - "const": "FUNNELS", - "type": "string" + "$ref": "#/definitions/FunnelsQueryResponse" }, - "results": { + "probability": { "additionalProperties": { - "$ref": "#/definitions/ExperimentVariantFunnelResult" + "type": "number" }, "type": "object" + }, + "significance_code": { + "$ref": "#/definitions/ExperimentSignificanceCode" + }, + "significant": { + "type": "boolean" + }, + "variants": { + "items": { + "$ref": "#/definitions/ExperimentVariantFunnelsBaseStats" + }, + "type": "array" } }, - "required": ["insight", "results"], + "required": [ + "insight", + "variants", + "probability", + "significant", + "significance_code", + "expected_loss", + "credible_intervals" + ], "type": "object" }, { @@ -8834,7 +8963,7 @@ }, "variants": { "items": { - "$ref": "#/definitions/ExperimentVariantTrendBaseStats" + "$ref": "#/definitions/ExperimentVariantTrendsBaseStats" }, "type": "array" } @@ -9398,18 +9527,51 @@ { "additionalProperties": false, "properties": { + "credible_intervals": { + "additionalProperties": { + "items": { + "type": "number" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + "type": "object" + }, + "expected_loss": { + "type": "number" + }, "insight": { - "const": "FUNNELS", - "type": "string" + "$ref": "#/definitions/FunnelsQueryResponse" }, - "results": { + "probability": { "additionalProperties": { - "$ref": "#/definitions/ExperimentVariantFunnelResult" + "type": "number" }, "type": "object" + }, + "significance_code": { + "$ref": "#/definitions/ExperimentSignificanceCode" + }, + "significant": { + "type": "boolean" + }, + "variants": { + "items": { + "$ref": "#/definitions/ExperimentVariantFunnelsBaseStats" + }, + "type": "array" } }, - "required": ["insight", "results"], + "required": [ + "credible_intervals", + "expected_loss", + "insight", + "probability", + "significance_code", + "significant", + "variants" + ], "type": "object" }, { @@ -9446,7 +9608,7 @@ }, "variants": { "items": { - "$ref": "#/definitions/ExperimentVariantTrendBaseStats" + "$ref": "#/definitions/ExperimentVariantTrendsBaseStats" }, "type": "array" } @@ -9839,10 +10001,10 @@ "$ref": "#/definitions/ErrorTrackingQuery" }, { - "$ref": "#/definitions/ExperimentFunnelQuery" + "$ref": "#/definitions/ExperimentFunnelsQuery" }, { - "$ref": "#/definitions/ExperimentTrendQuery" + "$ref": "#/definitions/ExperimentTrendsQuery" }, { "$ref": "#/definitions/DataVisualizationNode" diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 17e84a783f3e9..1887f57ee0f96 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -100,8 +100,8 @@ export enum NodeKind { WebGoalsQuery = 'WebGoalsQuery', // Experiment queries - ExperimentFunnelQuery = 'ExperimentFunnelQuery', - ExperimentTrendQuery = 'ExperimentTrendQuery', + ExperimentFunnelsQuery = 'ExperimentFunnelsQuery', + ExperimentTrendsQuery = 'ExperimentTrendsQuery', // Database metadata DatabaseSchemaQuery = 'DatabaseSchemaQuery', @@ -133,8 +133,8 @@ export type AnyDataNode = | WebGoalsQuery | SessionAttributionExplorerQuery | ErrorTrackingQuery - | ExperimentFunnelQuery - | ExperimentTrendQuery + | ExperimentFunnelsQuery + | ExperimentTrendsQuery /** * @discriminator kind @@ -161,8 +161,8 @@ export type QuerySchema = | WebGoalsQuery | SessionAttributionExplorerQuery | ErrorTrackingQuery - | ExperimentFunnelQuery - | ExperimentTrendQuery + | ExperimentFunnelsQuery + | ExperimentTrendsQuery // Interface nodes | DataVisualizationNode @@ -610,8 +610,8 @@ export interface DataTableNode | WebGoalsQuery | SessionAttributionExplorerQuery | ErrorTrackingQuery - | ExperimentFunnelQuery - | ExperimentTrendQuery + | ExperimentFunnelsQuery + | ExperimentTrendsQuery )['response'] > >, @@ -631,8 +631,8 @@ export interface DataTableNode | WebGoalsQuery | SessionAttributionExplorerQuery | ErrorTrackingQuery - | ExperimentFunnelQuery - | ExperimentTrendQuery + | ExperimentFunnelsQuery + | ExperimentTrendsQuery /** Columns shown in the table, unless the `source` provides them. */ columns?: HogQLExpression[] /** Columns that aren't shown in the table, even if in columns or returned data */ @@ -1596,14 +1596,14 @@ export type InsightQueryNode = | StickinessQuery | LifecycleQuery -export interface ExperimentVariantTrendBaseStats { +export interface ExperimentVariantTrendsBaseStats { key: string count: number exposure: number absolute_exposure: number } -export interface ExperimentVariantFunnelResult { +export interface ExperimentVariantFunnelsBaseStats { key: string success_count: number failure_count: number @@ -1617,9 +1617,9 @@ export enum ExperimentSignificanceCode { HighPValue = 'high_p_value', } -export interface ExperimentTrendQueryResponse { +export interface ExperimentTrendsQueryResponse { insight: TrendsQueryResponse - variants: ExperimentVariantTrendBaseStats[] + variants: ExperimentVariantTrendsBaseStats[] probability: Record significant: boolean significance_code: ExperimentSignificanceCode @@ -1627,26 +1627,31 @@ export interface ExperimentTrendQueryResponse { credible_intervals: Record } -export type CachedExperimentTrendQueryResponse = CachedQueryResponse +export type CachedExperimentTrendsQueryResponse = CachedQueryResponse -export interface ExperimentFunnelQueryResponse { - insight: InsightType.FUNNELS - results: Record +export interface ExperimentFunnelsQueryResponse { + insight: FunnelsQueryResponse + variants: ExperimentVariantFunnelsBaseStats[] + probability: Record + significant: boolean + significance_code: ExperimentSignificanceCode + expected_loss: number + credible_intervals: Record } -export type CachedExperimentFunnelQueryResponse = CachedQueryResponse +export type CachedExperimentFunnelsQueryResponse = CachedQueryResponse -export interface ExperimentFunnelQuery extends DataNode { - kind: NodeKind.ExperimentFunnelQuery +export interface ExperimentFunnelsQuery extends DataNode { + kind: NodeKind.ExperimentFunnelsQuery source: FunnelsQuery experiment_id: integer } -export interface ExperimentTrendQuery extends DataNode { - kind: NodeKind.ExperimentTrendQuery +export interface ExperimentTrendsQuery extends DataNode { + kind: NodeKind.ExperimentTrendsQuery count_query: TrendsQuery // Defaults to $feature_flag_called if not specified - // https://github.com/PostHog/posthog/blob/master/posthog/hogql_queries/experiments/experiment_trend_query_runner.py + // https://github.com/PostHog/posthog/blob/master/posthog/hogql_queries/experiments/experiment_trends_query_runner.py exposure_query?: TrendsQuery experiment_id: integer } diff --git a/frontend/src/scenes/saved-insights/SavedInsights.tsx b/frontend/src/scenes/saved-insights/SavedInsights.tsx index 4810a04db4bdf..554f0be7f5c45 100644 --- a/frontend/src/scenes/saved-insights/SavedInsights.tsx +++ b/frontend/src/scenes/saved-insights/SavedInsights.tsx @@ -338,14 +338,14 @@ export const QUERY_TYPES_METADATA: Record = { icon: IconVideoCamera, inMenu: false, }, - [NodeKind.ExperimentTrendQuery]: { - name: 'Experiment Result', + [NodeKind.ExperimentTrendsQuery]: { + name: 'Experiment Trends Result', description: 'View experiment trend result', icon: IconFlask, inMenu: false, }, - [NodeKind.ExperimentFunnelQuery]: { - name: 'Experiment Funnel', + [NodeKind.ExperimentFunnelsQuery]: { + name: 'Experiment Funnels Result', description: 'View experiment funnel result', icon: IconFlask, inMenu: false, diff --git a/posthog/hogql_queries/experiments/experiment_funnel_query_runner.py b/posthog/hogql_queries/experiments/experiment_funnel_query_runner.py deleted file mode 100644 index 0ff9a1058977b..0000000000000 --- a/posthog/hogql_queries/experiments/experiment_funnel_query_runner.py +++ /dev/null @@ -1,93 +0,0 @@ -from posthog.hogql import ast -from posthog.hogql_queries.query_runner import QueryRunner -from posthog.models.experiment import Experiment -from ..insights.funnels.funnels_query_runner import FunnelsQueryRunner -from posthog.schema import ( - CachedExperimentFunnelQueryResponse, - ExperimentFunnelQuery, - ExperimentFunnelQueryResponse, - ExperimentVariantFunnelResult, - FunnelsQuery, - InsightDateRange, - BreakdownFilter, -) -from typing import Any -from zoneinfo import ZoneInfo - - -class ExperimentFunnelQueryRunner(QueryRunner): - query: ExperimentFunnelQuery - response: ExperimentFunnelQueryResponse - cached_response: CachedExperimentFunnelQueryResponse - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.experiment = Experiment.objects.get(id=self.query.experiment_id) - self.feature_flag = self.experiment.feature_flag - self.prepared_funnel_query = self._prepare_funnel_query() - self.query_runner = FunnelsQueryRunner( - query=self.prepared_funnel_query, team=self.team, timings=self.timings, limit_context=self.limit_context - ) - - def calculate(self) -> ExperimentFunnelQueryResponse: - response = self.query_runner.calculate() - results = self._process_results(response.results) - return ExperimentFunnelQueryResponse(insight="FUNNELS", results=results) - - def _prepare_funnel_query(self) -> FunnelsQuery: - """ - This method takes the raw funnel query and adapts it - for the needs of experiment analysis: - - 1. Set the date range to match the experiment's duration, using the project's timezone. - 2. Configure the breakdown to use the feature flag key, which allows us - to separate results for different experiment variants. - """ - # Clone the source query - prepared_funnel_query = FunnelsQuery(**self.query.source.model_dump()) - - # Set the date range to match the experiment's duration, using the project's timezone - if self.team.timezone: - tz = ZoneInfo(self.team.timezone) - start_date = self.experiment.start_date.astimezone(tz) if self.experiment.start_date else None - end_date = self.experiment.end_date.astimezone(tz) if self.experiment.end_date else None - else: - start_date = self.experiment.start_date - end_date = self.experiment.end_date - - prepared_funnel_query.dateRange = InsightDateRange( - date_from=start_date.isoformat() if start_date else None, - date_to=end_date.isoformat() if end_date else None, - explicitDate=True, - ) - - # Configure the breakdown to use the feature flag key - prepared_funnel_query.breakdownFilter = BreakdownFilter( - breakdown=f"$feature/{self.feature_flag.key}", - breakdown_type="event", - ) - - return prepared_funnel_query - - def _process_results(self, funnels_results: list[list[dict[str, Any]]]) -> dict[str, ExperimentVariantFunnelResult]: - variants = self.feature_flag.variants - processed_results = { - variant["key"]: ExperimentVariantFunnelResult(key=variant["key"], success_count=0, failure_count=0) - for variant in variants - } - - for result in funnels_results: - first_step = result[0] - last_step = result[-1] - variant = first_step.get("breakdown_value") - variant_str = variant[0] if isinstance(variant, list) else str(variant) - if variant_str in processed_results: - total_count = first_step.get("count", 0) - success_count = last_step.get("count", 0) if len(result) > 1 else 0 - processed_results[variant_str].success_count = success_count - processed_results[variant_str].failure_count = total_count - success_count - - return processed_results - - def to_query(self) -> ast.SelectQuery: - raise ValueError(f"Cannot convert source query of type {self.query.source.kind} to query") diff --git a/posthog/hogql_queries/experiments/experiment_funnels_query_runner.py b/posthog/hogql_queries/experiments/experiment_funnels_query_runner.py new file mode 100644 index 0000000000000..c6783daa489e0 --- /dev/null +++ b/posthog/hogql_queries/experiments/experiment_funnels_query_runner.py @@ -0,0 +1,181 @@ +import json +from posthog.constants import ExperimentNoResultsErrorKeys +from posthog.hogql import ast +from posthog.hogql_queries.experiments import CONTROL_VARIANT_KEY +from posthog.hogql_queries.experiments.funnels_statistics import ( + are_results_significant, + calculate_credible_intervals, + calculate_probabilities, +) +from posthog.hogql_queries.query_runner import QueryRunner +from posthog.models.experiment import Experiment +from ..insights.funnels.funnels_query_runner import FunnelsQueryRunner +from posthog.schema import ( + CachedExperimentFunnelsQueryResponse, + ExperimentFunnelsQuery, + ExperimentFunnelsQueryResponse, + ExperimentSignificanceCode, + ExperimentVariantFunnelsBaseStats, + FunnelsQuery, + FunnelsQueryResponse, + InsightDateRange, + BreakdownFilter, +) +from typing import Optional, Any, cast +from zoneinfo import ZoneInfo +from rest_framework.exceptions import ValidationError + + +class ExperimentFunnelsQueryRunner(QueryRunner): + query: ExperimentFunnelsQuery + response: ExperimentFunnelsQueryResponse + cached_response: CachedExperimentFunnelsQueryResponse + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.experiment = Experiment.objects.get(id=self.query.experiment_id) + self.feature_flag = self.experiment.feature_flag + self.variants = [variant["key"] for variant in self.feature_flag.variants] + self.prepared_funnel_query = self._prepare_funnel_query() + self.funnels_query_runner = FunnelsQueryRunner( + query=self.prepared_funnel_query, team=self.team, timings=self.timings, limit_context=self.limit_context + ) + + def calculate(self) -> ExperimentFunnelsQueryResponse: + funnels_result = self.funnels_query_runner.calculate() + + self._validate_event_variants(funnels_result) + + # Statistical analysis + control_variant, test_variants = self._get_variants_with_base_stats(funnels_result) + probabilities = calculate_probabilities(control_variant, test_variants) + significance_code, loss = are_results_significant(control_variant, test_variants, probabilities) + credible_intervals = calculate_credible_intervals([control_variant, *test_variants]) + + return ExperimentFunnelsQueryResponse( + insight=funnels_result, + variants=[variant.model_dump() for variant in [control_variant, *test_variants]], + probability={ + variant.key: probability + for variant, probability in zip([control_variant, *test_variants], probabilities) + }, + significant=significance_code == ExperimentSignificanceCode.SIGNIFICANT, + significance_code=significance_code, + expected_loss=loss, + credible_intervals=credible_intervals, + ) + + def _prepare_funnel_query(self) -> FunnelsQuery: + """ + This method takes the raw funnel query and adapts it + for the needs of experiment analysis: + + 1. Set the date range to match the experiment's duration, using the project's timezone. + 2. Configure the breakdown to use the feature flag key, which allows us + to separate results for different experiment variants. + """ + # Clone the source query + prepared_funnel_query = FunnelsQuery(**self.query.source.model_dump()) + + # Set the date range to match the experiment's duration, using the project's timezone + if self.team.timezone: + tz = ZoneInfo(self.team.timezone) + start_date = self.experiment.start_date.astimezone(tz) if self.experiment.start_date else None + end_date = self.experiment.end_date.astimezone(tz) if self.experiment.end_date else None + else: + start_date = self.experiment.start_date + end_date = self.experiment.end_date + + prepared_funnel_query.dateRange = InsightDateRange( + date_from=start_date.isoformat() if start_date else None, + date_to=end_date.isoformat() if end_date else None, + explicitDate=True, + ) + + # Configure the breakdown to use the feature flag key + prepared_funnel_query.breakdownFilter = BreakdownFilter( + breakdown=f"$feature/{self.feature_flag.key}", + breakdown_type="event", + ) + + return prepared_funnel_query + + def _get_variants_with_base_stats( + self, funnels_result: FunnelsQueryResponse + ) -> tuple[ExperimentVariantFunnelsBaseStats, list[ExperimentVariantFunnelsBaseStats]]: + control_variant: Optional[ExperimentVariantFunnelsBaseStats] = None + test_variants = [] + + for result in funnels_result.results: + result_dict = cast(list[dict[str, Any]], result) + first_step = result_dict[0] + last_step = result_dict[-1] + + total = first_step.get("count", 0) + success = last_step.get("count", 0) if len(result_dict) > 1 else 0 + failure = total - success + + breakdown_value = cast(list[str], first_step["breakdown_value"])[0] + + if breakdown_value == CONTROL_VARIANT_KEY: + control_variant = ExperimentVariantFunnelsBaseStats( + key=breakdown_value, + success_count=int(success), + failure_count=int(failure), + ) + else: + test_variants.append( + ExperimentVariantFunnelsBaseStats( + key=breakdown_value, success_count=int(success), failure_count=int(failure) + ) + ) + + if control_variant is None: + raise ValueError("Control variant not found in count results") + + return control_variant, test_variants + + def _validate_event_variants(self, funnels_result: FunnelsQueryResponse): + errors = { + ExperimentNoResultsErrorKeys.NO_EVENTS: True, + ExperimentNoResultsErrorKeys.NO_FLAG_INFO: True, + ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT: True, + ExperimentNoResultsErrorKeys.NO_TEST_VARIANT: True, + } + + if not funnels_result.results or not funnels_result.results: + raise ValidationError(code="no-results", detail=json.dumps(errors)) + + errors[ExperimentNoResultsErrorKeys.NO_EVENTS] = False + + # Funnels: the first step must be present for *any* results to show up + eventsWithOrderZero = [] + for eventArr in funnels_result.results: + for event in eventArr: + event_dict = cast(dict[str, Any], event) + if event_dict.get("order") == 0: + eventsWithOrderZero.append(event_dict) + + # Check if "control" is present + for event in eventsWithOrderZero: + event_variant = event.get("breakdown_value", [None])[0] + if event_variant == "control": + errors[ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT] = False + errors[ExperimentNoResultsErrorKeys.NO_FLAG_INFO] = False + break + + # Check if at least one of the test variants is present + test_variants = [variant for variant in self.variants if variant != "control"] + for event in eventsWithOrderZero: + event_variant = event.get("breakdown_value", [None])[0] + if event_variant in test_variants: + errors[ExperimentNoResultsErrorKeys.NO_TEST_VARIANT] = False + errors[ExperimentNoResultsErrorKeys.NO_FLAG_INFO] = False + break + + has_errors = any(errors.values()) + if has_errors: + raise ValidationError(detail=json.dumps(errors)) + + def to_query(self) -> ast.SelectQuery: + raise ValueError(f"Cannot convert source query of type {self.query.source.kind} to query") diff --git a/posthog/hogql_queries/experiments/experiment_trend_query_runner.py b/posthog/hogql_queries/experiments/experiment_trends_query_runner.py similarity index 93% rename from posthog/hogql_queries/experiments/experiment_trend_query_runner.py rename to posthog/hogql_queries/experiments/experiment_trends_query_runner.py index 71173cd35f9f6..7389b65a29bf6 100644 --- a/posthog/hogql_queries/experiments/experiment_trend_query_runner.py +++ b/posthog/hogql_queries/experiments/experiment_trends_query_runner.py @@ -4,7 +4,7 @@ from posthog.constants import ExperimentNoResultsErrorKeys from posthog.hogql import ast from posthog.hogql_queries.experiments import CONTROL_VARIANT_KEY -from posthog.hogql_queries.experiments.trend_statistics import ( +from posthog.hogql_queries.experiments.trends_statistics import ( are_results_significant, calculate_credible_intervals, calculate_probabilities, @@ -17,14 +17,14 @@ from posthog.schema import ( BaseMathType, BreakdownFilter, - CachedExperimentTrendQueryResponse, + CachedExperimentTrendsQueryResponse, ChartDisplayType, EventPropertyFilter, EventsNode, ExperimentSignificanceCode, - ExperimentTrendQuery, - ExperimentTrendQueryResponse, - ExperimentVariantTrendBaseStats, + ExperimentTrendsQuery, + ExperimentTrendsQueryResponse, + ExperimentVariantTrendsBaseStats, InsightDateRange, PropertyMathType, TrendsFilter, @@ -35,10 +35,10 @@ import threading -class ExperimentTrendQueryRunner(QueryRunner): - query: ExperimentTrendQuery - response: ExperimentTrendQueryResponse - cached_response: CachedExperimentTrendQueryResponse +class ExperimentTrendsQueryRunner(QueryRunner): + query: ExperimentTrendsQuery + response: ExperimentTrendsQueryResponse + cached_response: CachedExperimentTrendsQueryResponse def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -207,7 +207,7 @@ def _prepare_exposure_query(self) -> TrendsQuery: return prepared_exposure_query - def calculate(self) -> ExperimentTrendQueryResponse: + def calculate(self) -> ExperimentTrendsQueryResponse: shared_results: dict[str, Optional[Any]] = {"count_result": None, "exposure_result": None} errors = [] @@ -249,14 +249,12 @@ def run(query_runner: TrendsQueryRunner, result_key: str, is_parallel: bool): self._validate_event_variants(count_result) # Statistical analysis - control_variant, test_variants = self._get_variants_with_base_stats( - count_result.results, exposure_result.results - ) + control_variant, test_variants = self._get_variants_with_base_stats(count_result, exposure_result) probabilities = calculate_probabilities(control_variant, test_variants) significance_code, p_value = are_results_significant(control_variant, test_variants, probabilities) credible_intervals = calculate_credible_intervals([control_variant, *test_variants]) - return ExperimentTrendQueryResponse( + return ExperimentTrendsQueryResponse( insight=count_result, variants=[variant.model_dump() for variant in [control_variant, *test_variants]], probability={ @@ -270,14 +268,14 @@ def run(query_runner: TrendsQueryRunner, result_key: str, is_parallel: bool): ) def _get_variants_with_base_stats( - self, count_results: list[dict[str, Any]], exposure_results: list[dict[str, Any]] - ) -> tuple[ExperimentVariantTrendBaseStats, list[ExperimentVariantTrendBaseStats]]: - control_variant: Optional[ExperimentVariantTrendBaseStats] = None + self, count_results: TrendsQueryResponse, exposure_results: TrendsQueryResponse + ) -> tuple[ExperimentVariantTrendsBaseStats, list[ExperimentVariantTrendsBaseStats]]: + control_variant: Optional[ExperimentVariantTrendsBaseStats] = None test_variants = [] exposure_counts = {} exposure_ratios = {} - for result in exposure_results: + for result in exposure_results.results: count = result.get("count", 0) breakdown_value = result.get("breakdown_value") exposure_counts[breakdown_value] = count @@ -288,11 +286,11 @@ def _get_variants_with_base_stats( for key, count in exposure_counts.items(): exposure_ratios[key] = count / control_exposure - for result in count_results: + for result in count_results.results: count = result.get("count", 0) breakdown_value = result.get("breakdown_value") if breakdown_value == CONTROL_VARIANT_KEY: - control_variant = ExperimentVariantTrendBaseStats( + control_variant = ExperimentVariantTrendsBaseStats( key=breakdown_value, count=count, exposure=1, @@ -301,7 +299,7 @@ def _get_variants_with_base_stats( ) else: test_variants.append( - ExperimentVariantTrendBaseStats( + ExperimentVariantTrendsBaseStats( key=breakdown_value, count=count, # TODO: in the absence of exposure data, we should throw rather than default to 1 diff --git a/posthog/hogql_queries/experiments/funnel_statistics.py b/posthog/hogql_queries/experiments/funnels_statistics.py similarity index 93% rename from posthog/hogql_queries/experiments/funnel_statistics.py rename to posthog/hogql_queries/experiments/funnels_statistics.py index 0c84714ae3842..cdec48fa3c681 100644 --- a/posthog/hogql_queries/experiments/funnel_statistics.py +++ b/posthog/hogql_queries/experiments/funnels_statistics.py @@ -7,14 +7,14 @@ FF_DISTRIBUTION_THRESHOLD, MIN_PROBABILITY_FOR_SIGNIFICANCE, ) -from posthog.schema import ExperimentSignificanceCode, ExperimentVariantFunnelResult +from posthog.schema import ExperimentSignificanceCode, ExperimentVariantFunnelsBaseStats Probability = float def calculate_probabilities( - control_variant: ExperimentVariantFunnelResult, - test_variants: list[ExperimentVariantFunnelResult], + control_variant: ExperimentVariantFunnelsBaseStats, + test_variants: list[ExperimentVariantFunnelsBaseStats], priors: tuple[int, int] = (1, 1), ) -> list[Probability]: """ @@ -60,7 +60,7 @@ def calculate_probabilities( def simulate_winning_variant_for_conversion( - target_variant: ExperimentVariantFunnelResult, variants: list[ExperimentVariantFunnelResult] + target_variant: ExperimentVariantFunnelsBaseStats, variants: list[ExperimentVariantFunnelsBaseStats] ) -> Probability: random_sampler = default_rng() prior_success = 1 @@ -94,11 +94,11 @@ def simulate_winning_variant_for_conversion( def are_results_significant( - control_variant: ExperimentVariantFunnelResult, - test_variants: list[ExperimentVariantFunnelResult], + control_variant: ExperimentVariantFunnelsBaseStats, + test_variants: list[ExperimentVariantFunnelsBaseStats], probabilities: list[Probability], ) -> tuple[ExperimentSignificanceCode, Probability]: - def get_conversion_rate(variant: ExperimentVariantFunnelResult): + def get_conversion_rate(variant: ExperimentVariantFunnelsBaseStats): return variant.success_count / (variant.success_count + variant.failure_count) control_sample_size = control_variant.success_count + control_variant.failure_count @@ -136,7 +136,7 @@ def get_conversion_rate(variant: ExperimentVariantFunnelResult): def calculate_expected_loss( - target_variant: ExperimentVariantFunnelResult, variants: list[ExperimentVariantFunnelResult] + target_variant: ExperimentVariantFunnelsBaseStats, variants: list[ExperimentVariantFunnelsBaseStats] ) -> float: """ Calculates expected loss in conversion rate for a given variant. diff --git a/posthog/hogql_queries/experiments/test/test_experiment_funnel_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_funnel_query_runner.py deleted file mode 100644 index 7d1472d29315a..0000000000000 --- a/posthog/hogql_queries/experiments/test/test_experiment_funnel_query_runner.py +++ /dev/null @@ -1,107 +0,0 @@ -from posthog.hogql_queries.experiments.experiment_funnel_query_runner import ExperimentFunnelQueryRunner -from posthog.models.experiment import Experiment -from posthog.models.feature_flag.feature_flag import FeatureFlag -from posthog.schema import ( - EventsNode, - ExperimentFunnelQuery, - ExperimentFunnelQueryResponse, - FunnelsQuery, -) -from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events -from freezegun import freeze_time -from typing import cast -from django.utils import timezone -from datetime import timedelta - - -class TestExperimentFunnelQueryRunner(ClickhouseTestMixin, APIBaseTest): - @freeze_time("2020-01-01T12:00:00Z") - def test_query_runner(self): - feature_flag = FeatureFlag.objects.create( - name="Test experiment flag", - key="test-experiment", - team=self.team, - filters={ - "groups": [{"properties": [], "rollout_percentage": None}], - "multivariate": { - "variants": [ - { - "key": "control", - "name": "Control", - "rollout_percentage": 50, - }, - { - "key": "test", - "name": "Test", - "rollout_percentage": 50, - }, - ] - }, - }, - created_by=self.user, - ) - - experiment = Experiment.objects.create( - name="test-experiment", - team=self.team, - feature_flag=feature_flag, - start_date=timezone.now(), - end_date=timezone.now() + timedelta(days=14), - ) - - feature_flag_property = f"$feature/{feature_flag.key}" - - funnels_query = FunnelsQuery( - series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], - dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, - ) - experiment_query = ExperimentFunnelQuery( - experiment_id=experiment.id, - kind="ExperimentFunnelQuery", - source=funnels_query, - ) - - experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] - experiment.save() - - for variant, purchase_count in [("control", 6), ("test", 8)]: - for i in range(10): - _create_person(distinct_ids=[f"user_{variant}_{i}"], team_id=self.team.pk) - _create_event( - team=self.team, - event="$pageview", - distinct_id=f"user_{variant}_{i}", - timestamp="2020-01-02T12:00:00Z", - properties={feature_flag_property: variant}, - ) - if i < purchase_count: - _create_event( - team=self.team, - event="purchase", - distinct_id=f"user_{variant}_{i}", - timestamp="2020-01-02T12:01:00Z", - properties={feature_flag_property: variant}, - ) - - flush_persons_and_events() - - query_runner = ExperimentFunnelQueryRunner( - query=ExperimentFunnelQuery(**experiment.metrics[0]["query"]), team=self.team - ) - result = query_runner.calculate() - - self.assertEqual(result.insight, "FUNNELS") - self.assertEqual(len(result.results), 2) - - funnel_result = cast(ExperimentFunnelQueryResponse, result) - - self.assertIn("control", funnel_result.results) - self.assertIn("test", funnel_result.results) - - control_result = funnel_result.results["control"] - test_result = funnel_result.results["test"] - - self.assertEqual(control_result.success_count, 6) - self.assertEqual(control_result.failure_count, 4) - self.assertEqual(test_result.success_count, 8) - self.assertEqual(test_result.failure_count, 2) diff --git a/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py new file mode 100644 index 0000000000000..005fe82e089ae --- /dev/null +++ b/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py @@ -0,0 +1,359 @@ +from typing import cast +from posthog.hogql_queries.experiments.experiment_funnels_query_runner import ExperimentFunnelsQueryRunner +from posthog.models.experiment import Experiment +from posthog.models.feature_flag.feature_flag import FeatureFlag +from posthog.schema import ( + EventsNode, + ExperimentFunnelsQuery, + ExperimentSignificanceCode, + FunnelsQuery, +) +from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events +from freezegun import freeze_time +from django.utils import timezone +from datetime import timedelta +from rest_framework.exceptions import ValidationError +from posthog.constants import ExperimentNoResultsErrorKeys +import json +from posthog.test.test_journeys import journeys_for + + +class TestExperimentFunnelsQueryRunner(ClickhouseTestMixin, APIBaseTest): + def create_feature_flag(self, key="test-experiment"): + return FeatureFlag.objects.create( + name=f"Test experiment flag: {key}", + key=key, + team=self.team, + filters={ + "groups": [{"properties": [], "rollout_percentage": None}], + "multivariate": { + "variants": [ + { + "key": "control", + "name": "Control", + "rollout_percentage": 50, + }, + { + "key": "test", + "name": "Test", + "rollout_percentage": 50, + }, + ] + }, + }, + created_by=self.user, + ) + + def create_experiment(self, name="test-experiment", feature_flag=None): + if feature_flag is None: + feature_flag = self.create_feature_flag(name) + return Experiment.objects.create( + name=name, + team=self.team, + feature_flag=feature_flag, + start_date=timezone.now(), + end_date=timezone.now() + timedelta(days=14), + ) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner(self): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag) + + feature_flag_property = f"$feature/{feature_flag.key}" + + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + ) + experiment_query = ExperimentFunnelsQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelsQuery", + source=funnels_query, + ) + + experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] + experiment.save() + + for variant, purchase_count in [("control", 6), ("test", 8)]: + for i in range(10): + _create_person(distinct_ids=[f"user_{variant}_{i}"], team_id=self.team.pk) + _create_event( + team=self.team, + event="$pageview", + distinct_id=f"user_{variant}_{i}", + timestamp="2020-01-02T12:00:00Z", + properties={feature_flag_property: variant}, + ) + if i < purchase_count: + _create_event( + team=self.team, + event="purchase", + distinct_id=f"user_{variant}_{i}", + timestamp="2020-01-02T12:01:00Z", + properties={feature_flag_property: variant}, + ) + + flush_persons_and_events() + + query_runner = ExperimentFunnelsQueryRunner( + query=ExperimentFunnelsQuery(**experiment.metrics[0]["query"]), team=self.team + ) + result = query_runner.calculate() + + self.assertEqual(len(result.variants), 2) + + control_variant = next(variant for variant in result.variants if variant.key == "control") + test_variant = next(variant for variant in result.variants if variant.key == "test") + + self.assertEqual(control_variant.success_count, 6) + self.assertEqual(control_variant.failure_count, 4) + self.assertEqual(test_variant.success_count, 8) + self.assertEqual(test_variant.failure_count, 2) + + self.assertIn("control", result.probability) + self.assertIn("test", result.probability) + + self.assertIn("control", result.credible_intervals) + self.assertIn("test", result.credible_intervals) + + @freeze_time("2020-01-01T12:00:00Z") + def test_query_runner_standard_flow(self): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag) + + ff_property = f"$feature/{feature_flag.key}" + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + ) + experiment_query = ExperimentFunnelsQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelsQuery", + source=funnels_query, + ) + + experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] + experiment.save() + + journeys_for( + { + "user_control_1": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "control"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "control"}}, + ], + "user_control_2": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "control"}}, + ], + "user_control_3": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "control"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "control"}}, + ], + "user_test_1": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, + ], + "user_test_2": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, + ], + "user_test_3": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + ], + "user_test_4": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, + ], + }, + self.team, + ) + + flush_persons_and_events() + + query_runner = ExperimentFunnelsQueryRunner( + query=ExperimentFunnelsQuery(**experiment.metrics[0]["query"]), team=self.team + ) + result = query_runner.calculate() + + self.assertEqual(len(result.variants), 2) + for variant in result.variants: + self.assertIn(variant.key, ["control", "test"]) + + control_variant = next(v for v in result.variants if v.key == "control") + test_variant = next(v for v in result.variants if v.key == "test") + + self.assertEqual(control_variant.success_count, 2) + self.assertEqual(control_variant.failure_count, 1) + self.assertEqual(test_variant.success_count, 3) + self.assertEqual(test_variant.failure_count, 1) + + self.assertAlmostEqual(result.probability["control"], 0.407, places=2) + self.assertAlmostEqual(result.probability["test"], 0.593, places=2) + + self.assertAlmostEqual(result.credible_intervals["control"][0], 0.1941, places=3) + self.assertAlmostEqual(result.credible_intervals["control"][1], 0.9324, places=3) + self.assertAlmostEqual(result.credible_intervals["test"][0], 0.2836, places=3) + self.assertAlmostEqual(result.credible_intervals["test"][1], 0.9473, places=3) + + self.assertEqual(result.significance_code, ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE) + + self.assertFalse(result.significant) + self.assertEqual(len(result.variants), 2) + self.assertAlmostEqual(result.expected_loss, 1.0, places=1) + + @freeze_time("2020-01-01T12:00:00Z") + def test_validate_event_variants_no_events(self): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag) + + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + ) + experiment_query = ExperimentFunnelsQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelsQuery", + source=funnels_query, + ) + + query_runner = ExperimentFunnelsQueryRunner(query=experiment_query, team=self.team) + with self.assertRaises(ValidationError) as context: + query_runner.calculate() + + expected_errors = json.dumps( + { + ExperimentNoResultsErrorKeys.NO_EVENTS: True, + ExperimentNoResultsErrorKeys.NO_FLAG_INFO: True, + ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT: True, + ExperimentNoResultsErrorKeys.NO_TEST_VARIANT: True, + } + ) + self.assertEqual(cast(list, context.exception.detail)[0], expected_errors) + + @freeze_time("2020-01-01T12:00:00Z") + def test_validate_event_variants_no_control(self): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag) + + ff_property = f"$feature/{feature_flag.key}" + journeys_for( + { + "user_test": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "test"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "test"}}, + ], + }, + self.team, + ) + + flush_persons_and_events() + + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + ) + experiment_query = ExperimentFunnelsQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelsQuery", + source=funnels_query, + ) + + query_runner = ExperimentFunnelsQueryRunner(query=experiment_query, team=self.team) + with self.assertRaises(ValidationError) as context: + query_runner.calculate() + + expected_errors = json.dumps( + { + ExperimentNoResultsErrorKeys.NO_EVENTS: False, + ExperimentNoResultsErrorKeys.NO_FLAG_INFO: False, + ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT: True, + ExperimentNoResultsErrorKeys.NO_TEST_VARIANT: False, + } + ) + self.assertEqual(cast(list, context.exception.detail)[0], expected_errors) + + @freeze_time("2020-01-01T12:00:00Z") + def test_validate_event_variants_no_test(self): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag) + + ff_property = f"$feature/{feature_flag.key}" + journeys_for( + { + "user_control": [ + {"event": "$pageview", "timestamp": "2020-01-02", "properties": {ff_property: "control"}}, + {"event": "purchase", "timestamp": "2020-01-03", "properties": {ff_property: "control"}}, + ], + }, + self.team, + ) + + flush_persons_and_events() + + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + ) + experiment_query = ExperimentFunnelsQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelsQuery", + source=funnels_query, + ) + + query_runner = ExperimentFunnelsQueryRunner(query=experiment_query, team=self.team) + with self.assertRaises(ValidationError) as context: + query_runner.calculate() + + expected_errors = json.dumps( + { + ExperimentNoResultsErrorKeys.NO_EVENTS: False, + ExperimentNoResultsErrorKeys.NO_FLAG_INFO: False, + ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT: False, + ExperimentNoResultsErrorKeys.NO_TEST_VARIANT: True, + } + ) + self.assertEqual(cast(list, context.exception.detail)[0], expected_errors) + + @freeze_time("2020-01-01T12:00:00Z") + def test_validate_event_variants_no_flag_info(self): + feature_flag = self.create_feature_flag() + experiment = self.create_experiment(feature_flag=feature_flag) + + journeys_for( + { + "user_no_flag_1": [ + {"event": "$pageview", "timestamp": "2020-01-02"}, + {"event": "purchase", "timestamp": "2020-01-03"}, + ], + "user_no_flag_2": [ + {"event": "$pageview", "timestamp": "2020-01-03"}, + ], + }, + self.team, + ) + + flush_persons_and_events() + + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + ) + experiment_query = ExperimentFunnelsQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelsQuery", + source=funnels_query, + ) + + query_runner = ExperimentFunnelsQueryRunner(query=experiment_query, team=self.team) + with self.assertRaises(ValidationError) as context: + query_runner.calculate() + + expected_errors = json.dumps( + { + ExperimentNoResultsErrorKeys.NO_EVENTS: False, + ExperimentNoResultsErrorKeys.NO_FLAG_INFO: True, + ExperimentNoResultsErrorKeys.NO_CONTROL_VARIANT: True, + ExperimentNoResultsErrorKeys.NO_TEST_VARIANT: True, + } + ) + self.assertEqual(cast(list, context.exception.detail)[0], expected_errors) diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py similarity index 88% rename from posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py rename to posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py index 9d8484c6e25f3..bb3357e62232b 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py @@ -1,14 +1,13 @@ from django.test import override_settings -from posthog.hogql_queries.experiments.experiment_trend_query_runner import ExperimentTrendQueryRunner +from posthog.hogql_queries.experiments.experiment_trends_query_runner import ExperimentTrendsQueryRunner from posthog.models.experiment import Experiment from posthog.models.feature_flag.feature_flag import FeatureFlag from posthog.schema import ( EventsNode, ExperimentSignificanceCode, - ExperimentTrendQuery, - ExperimentTrendQueryResponse, + ExperimentTrendsQuery, + ExperimentTrendsQueryResponse, TrendsQuery, - TrendsQueryResponse, ) from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events from freezegun import freeze_time @@ -22,7 +21,7 @@ @override_settings(IN_UNIT_TESTING=True) -class TestExperimentTrendQueryRunner(ClickhouseTestMixin, APIBaseTest): +class TestExperimentTrendsQueryRunner(ClickhouseTestMixin, APIBaseTest): def create_feature_flag(self, key="test-experiment"): return FeatureFlag.objects.create( name=f"Test experiment flag: {key}", @@ -68,9 +67,9 @@ def test_query_runner(self): count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) exposure_query = TrendsQuery(series=[EventsNode(event="$feature_flag_called")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, exposure_query=exposure_query, ) @@ -100,8 +99,8 @@ def test_query_runner(self): flush_persons_and_events() - query_runner = ExperimentTrendQueryRunner( - query=ExperimentTrendQuery(**experiment.metrics[0]["query"]), team=self.team + query_runner = ExperimentTrendsQueryRunner( + query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team ) result = query_runner.calculate() @@ -126,9 +125,9 @@ def test_query_runner_with_custom_exposure(self): series=[EventsNode(event="custom_exposure_event", properties=[{"key": "valid_exposure", "value": "true"}])] ) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, exposure_query=exposure_query, ) @@ -198,12 +197,12 @@ def test_query_runner_with_custom_exposure(self): flush_persons_and_events() - query_runner = ExperimentTrendQueryRunner( - query=ExperimentTrendQuery(**experiment.metrics[0]["query"]), team=self.team + query_runner = ExperimentTrendsQueryRunner( + query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team ) result = query_runner.calculate() - trend_result = cast(ExperimentTrendQueryResponse, result) + trend_result = cast(ExperimentTrendsQueryResponse, result) control_result = next(variant for variant in trend_result.variants if variant.key == "control") test_result = next(variant for variant in trend_result.variants if variant.key == "test") @@ -222,9 +221,9 @@ def test_query_runner_with_default_exposure(self): ff_property = f"$feature/{feature_flag.key}" count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, exposure_query=None, # No exposure query provided ) @@ -290,12 +289,12 @@ def test_query_runner_with_default_exposure(self): flush_persons_and_events() - query_runner = ExperimentTrendQueryRunner( - query=ExperimentTrendQuery(**experiment.metrics[0]["query"]), team=self.team + query_runner = ExperimentTrendsQueryRunner( + query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team ) result = query_runner.calculate() - trend_result = cast(ExperimentTrendQueryResponse, result) + trend_result = cast(ExperimentTrendsQueryResponse, result) control_result = next(variant for variant in trend_result.variants if variant.key == "control") test_result = next(variant for variant in trend_result.variants if variant.key == "test") @@ -314,9 +313,9 @@ def test_query_runner_with_avg_math(self): count_query = TrendsQuery(series=[EventsNode(event="$pageview", math="avg")]) exposure_query = TrendsQuery(series=[EventsNode(event="$feature_flag_called")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, exposure_query=exposure_query, ) @@ -324,8 +323,8 @@ def test_query_runner_with_avg_math(self): experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] experiment.save() - query_runner = ExperimentTrendQueryRunner( - query=ExperimentTrendQuery(**experiment.metrics[0]["query"]), team=self.team + query_runner = ExperimentTrendsQueryRunner( + query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team ) prepared_count_query = query_runner.prepared_count_query @@ -340,9 +339,9 @@ def test_query_runner_standard_flow(self): count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) exposure_query = TrendsQuery(series=[EventsNode(event="$feature_flag_called")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, exposure_query=exposure_query, ) @@ -386,8 +385,8 @@ def test_query_runner_standard_flow(self): flush_persons_and_events() - query_runner = ExperimentTrendQueryRunner( - query=ExperimentTrendQuery(**experiment.metrics[0]["query"]), team=self.team + query_runner = ExperimentTrendsQueryRunner( + query=ExperimentTrendsQuery(**experiment.metrics[0]["query"]), team=self.team ) result = query_runner.calculate() @@ -433,16 +432,15 @@ def test_validate_event_variants_no_events(self): experiment = self.create_experiment(feature_flag=feature_flag) count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, ) - query_runner = ExperimentTrendQueryRunner(query=experiment_query, team=self.team) - + query_runner = ExperimentTrendsQueryRunner(query=experiment_query, team=self.team) with self.assertRaises(ValidationError) as context: - query_runner._validate_event_variants(TrendsQueryResponse(results=[])) + query_runner.calculate() expected_errors = json.dumps( { @@ -472,17 +470,15 @@ def test_validate_event_variants_no_control(self): flush_persons_and_events() count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, ) - query_runner = ExperimentTrendQueryRunner(query=experiment_query, team=self.team) - result = query_runner.count_query_runner.calculate() - + query_runner = ExperimentTrendsQueryRunner(query=experiment_query, team=self.team) with self.assertRaises(ValidationError) as context: - query_runner._validate_event_variants(result) + query_runner.calculate() expected_errors = json.dumps( { @@ -512,17 +508,15 @@ def test_validate_event_variants_no_test(self): flush_persons_and_events() count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, ) - query_runner = ExperimentTrendQueryRunner(query=experiment_query, team=self.team) - result = query_runner.count_query_runner.calculate() - + query_runner = ExperimentTrendsQueryRunner(query=experiment_query, team=self.team) with self.assertRaises(ValidationError) as context: - query_runner._validate_event_variants(result) + query_runner.calculate() expected_errors = json.dumps( { @@ -554,17 +548,15 @@ def test_validate_event_variants_no_flag_info(self): flush_persons_and_events() count_query = TrendsQuery(series=[EventsNode(event="$pageview")]) - experiment_query = ExperimentTrendQuery( + experiment_query = ExperimentTrendsQuery( experiment_id=experiment.id, - kind="ExperimentTrendQuery", + kind="ExperimentTrendsQuery", count_query=count_query, ) - query_runner = ExperimentTrendQueryRunner(query=experiment_query, team=self.team) - result = query_runner.count_query_runner.calculate() - + query_runner = ExperimentTrendsQueryRunner(query=experiment_query, team=self.team) with self.assertRaises(ValidationError) as context: - query_runner._validate_event_variants(result) + query_runner.calculate() expected_errors = json.dumps( { diff --git a/posthog/hogql_queries/experiments/trend_statistics.py b/posthog/hogql_queries/experiments/trends_statistics.py similarity index 93% rename from posthog/hogql_queries/experiments/trend_statistics.py rename to posthog/hogql_queries/experiments/trends_statistics.py index 9b2218267b0cc..61b19d1486f72 100644 --- a/posthog/hogql_queries/experiments/trend_statistics.py +++ b/posthog/hogql_queries/experiments/trends_statistics.py @@ -12,13 +12,13 @@ P_VALUE_SIGNIFICANCE_LEVEL, ) -from posthog.schema import ExperimentSignificanceCode, ExperimentVariantTrendBaseStats +from posthog.schema import ExperimentSignificanceCode, ExperimentVariantTrendsBaseStats Probability = float def calculate_probabilities( - control_variant: ExperimentVariantTrendBaseStats, test_variants: list[ExperimentVariantTrendBaseStats] + control_variant: ExperimentVariantTrendsBaseStats, test_variants: list[ExperimentVariantTrendsBaseStats] ) -> list[Probability]: """ Calculates probability that A is better than B. First variant is control, rest are test variants. @@ -59,7 +59,7 @@ def calculate_probabilities( def simulate_winning_variant_for_arrival_rates( - target_variant: ExperimentVariantTrendBaseStats, variants: list[ExperimentVariantTrendBaseStats] + target_variant: ExperimentVariantTrendsBaseStats, variants: list[ExperimentVariantTrendsBaseStats] ) -> float: random_sampler = default_rng() simulations_count = 100_000 @@ -85,8 +85,8 @@ def simulate_winning_variant_for_arrival_rates( def are_results_significant( - control_variant: ExperimentVariantTrendBaseStats, - test_variants: list[ExperimentVariantTrendBaseStats], + control_variant: ExperimentVariantTrendsBaseStats, + test_variants: list[ExperimentVariantTrendsBaseStats], probabilities: list[Probability], ) -> tuple[ExperimentSignificanceCode, Probability]: # TODO: Experiment with Expected Loss calculations for trend experiments @@ -152,7 +152,7 @@ def poisson_p_value(control_count, control_exposure, test_count, test_exposure): def calculate_p_value( - control_variant: ExperimentVariantTrendBaseStats, test_variants: list[ExperimentVariantTrendBaseStats] + control_variant: ExperimentVariantTrendsBaseStats, test_variants: list[ExperimentVariantTrendsBaseStats] ) -> Probability: best_test_variant = max(test_variants, key=lambda variant: variant.count) diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py index 2786a79f299aa..664430cc7da04 100644 --- a/posthog/hogql_queries/query_runner.py +++ b/posthog/hogql_queries/query_runner.py @@ -366,10 +366,10 @@ def get_query_runner( limit_context=limit_context, ) - if kind == "ExperimentFunnelQuery": - from .experiments.experiment_funnel_query_runner import ExperimentFunnelQueryRunner + if kind == "ExperimentFunnelsQuery": + from .experiments.experiment_funnels_query_runner import ExperimentFunnelsQueryRunner - return ExperimentFunnelQueryRunner( + return ExperimentFunnelsQueryRunner( query=query, team=team, timings=timings, @@ -377,10 +377,10 @@ def get_query_runner( limit_context=limit_context, ) - if kind == "ExperimentTrendQuery": - from .experiments.experiment_trend_query_runner import ExperimentTrendQueryRunner + if kind == "ExperimentTrendsQuery": + from .experiments.experiment_trends_query_runner import ExperimentTrendsQueryRunner - return ExperimentTrendQueryRunner( + return ExperimentTrendsQueryRunner( query=query, team=team, timings=timings, diff --git a/posthog/schema.py b/posthog/schema.py index afd2ca10dde3e..1ba777cd25b26 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -525,7 +525,7 @@ class ExperimentSignificanceCode(StrEnum): HIGH_P_VALUE = "high_p_value" -class ExperimentVariantFunnelResult(BaseModel): +class ExperimentVariantFunnelsBaseStats(BaseModel): model_config = ConfigDict( extra="forbid", ) @@ -534,7 +534,7 @@ class ExperimentVariantFunnelResult(BaseModel): success_count: float -class ExperimentVariantTrendBaseStats(BaseModel): +class ExperimentVariantTrendsBaseStats(BaseModel): model_config = ConfigDict( extra="forbid", ) @@ -875,8 +875,8 @@ class NodeKind(StrEnum): WEB_STATS_TABLE_QUERY = "WebStatsTableQuery" WEB_EXTERNAL_CLICKS_TABLE_QUERY = "WebExternalClicksTableQuery" WEB_GOALS_QUERY = "WebGoalsQuery" - EXPERIMENT_FUNNEL_QUERY = "ExperimentFunnelQuery" - EXPERIMENT_TREND_QUERY = "ExperimentTrendQuery" + EXPERIMENT_FUNNELS_QUERY = "ExperimentFunnelsQuery" + EXPERIMENT_TRENDS_QUERY = "ExperimentTrendsQuery" DATABASE_SCHEMA_QUERY = "DatabaseSchemaQuery" SUGGESTED_QUESTIONS_QUERY = "SuggestedQuestionsQuery" TEAM_TAXONOMY_QUERY = "TeamTaxonomyQuery" @@ -1027,14 +1027,6 @@ class QueryResponseAlternative7(BaseModel): warnings: list[HogQLNotice] -class QueryResponseAlternative16(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - insight: Literal["FUNNELS"] = "FUNNELS" - results: dict[str, ExperimentVariantFunnelResult] - - class QueryResponseAlternative38(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1922,27 +1914,7 @@ class CachedEventsQueryResponse(BaseModel): types: list[str] -class CachedExperimentFunnelQueryResponse(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - cache_key: str - cache_target_age: Optional[AwareDatetime] = None - calculation_trigger: Optional[str] = Field( - default=None, description="What triggered the calculation of the query, leave empty if user/immediate" - ) - insight: Literal["FUNNELS"] = "FUNNELS" - is_cached: bool - last_refresh: AwareDatetime - next_allowed_client_refresh: AwareDatetime - query_status: Optional[QueryStatus] = Field( - default=None, description="Query status indicates whether next to the provided data, a query is still running." - ) - results: dict[str, ExperimentVariantFunnelResult] - timezone: str - - -class CachedExperimentTrendQueryResponse(BaseModel): +class CachedExperimentTrendsQueryResponse(BaseModel): model_config = ConfigDict( extra="forbid", ) @@ -1964,7 +1936,7 @@ class CachedExperimentTrendQueryResponse(BaseModel): significance_code: ExperimentSignificanceCode significant: bool timezone: str - variants: list[ExperimentVariantTrendBaseStats] + variants: list[ExperimentVariantTrendsBaseStats] class CachedFunnelCorrelationResponse(BaseModel): @@ -2691,14 +2663,6 @@ class Response9(BaseModel): ) -class Response10(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - insight: Literal["FUNNELS"] = "FUNNELS" - results: dict[str, ExperimentVariantFunnelResult] - - class Response11(BaseModel): model_config = ConfigDict( extra="forbid", @@ -2709,7 +2673,7 @@ class Response11(BaseModel): probability: dict[str, float] significance_code: ExperimentSignificanceCode significant: bool - variants: list[ExperimentVariantTrendBaseStats] + variants: list[ExperimentVariantTrendsBaseStats] class DataWarehousePersonPropertyFilter(BaseModel): @@ -2862,15 +2826,7 @@ class EventsQueryResponse(BaseModel): types: list[str] -class ExperimentFunnelQueryResponse(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - insight: Literal["FUNNELS"] = "FUNNELS" - results: dict[str, ExperimentVariantFunnelResult] - - -class ExperimentTrendQueryResponse(BaseModel): +class ExperimentTrendsQueryResponse(BaseModel): model_config = ConfigDict( extra="forbid", ) @@ -2880,7 +2836,7 @@ class ExperimentTrendQueryResponse(BaseModel): probability: dict[str, float] significance_code: ExperimentSignificanceCode significant: bool - variants: list[ExperimentVariantTrendBaseStats] + variants: list[ExperimentVariantTrendsBaseStats] class BreakdownFilter1(BaseModel): @@ -3473,6 +3429,19 @@ class QueryResponseAlternative15(BaseModel): ) +class QueryResponseAlternative16(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + credible_intervals: dict[str, list[float]] + expected_loss: float + insight: FunnelsQueryResponse + probability: dict[str, float] + significance_code: ExperimentSignificanceCode + significant: bool + variants: list[ExperimentVariantFunnelsBaseStats] + + class QueryResponseAlternative17(BaseModel): model_config = ConfigDict( extra="forbid", @@ -3483,7 +3452,7 @@ class QueryResponseAlternative17(BaseModel): probability: dict[str, float] significance_code: ExperimentSignificanceCode significant: bool - variants: list[ExperimentVariantTrendBaseStats] + variants: list[ExperimentVariantTrendsBaseStats] class QueryResponseAlternative18(BaseModel): @@ -3722,6 +3691,19 @@ class QueryResponseAlternative27(BaseModel): ) +class QueryResponseAlternative28(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + credible_intervals: dict[str, list[float]] + expected_loss: float + insight: FunnelsQueryResponse + probability: dict[str, float] + significance_code: ExperimentSignificanceCode + significant: bool + variants: list[ExperimentVariantFunnelsBaseStats] + + class QueryResponseAlternative29(BaseModel): model_config = ConfigDict( extra="forbid", @@ -3732,7 +3714,7 @@ class QueryResponseAlternative29(BaseModel): probability: dict[str, float] significance_code: ExperimentSignificanceCode significant: bool - variants: list[ExperimentVariantTrendBaseStats] + variants: list[ExperimentVariantTrendsBaseStats] class QueryResponseAlternative30(BaseModel): @@ -4171,6 +4153,31 @@ class AnyResponseType( ] +class CachedExperimentFunnelsQueryResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + cache_key: str + cache_target_age: Optional[AwareDatetime] = None + calculation_trigger: Optional[str] = Field( + default=None, description="What triggered the calculation of the query, leave empty if user/immediate" + ) + credible_intervals: dict[str, list[float]] + expected_loss: float + insight: FunnelsQueryResponse + is_cached: bool + last_refresh: AwareDatetime + next_allowed_client_refresh: AwareDatetime + probability: dict[str, float] + query_status: Optional[QueryStatus] = Field( + default=None, description="Query status indicates whether next to the provided data, a query is still running." + ) + significance_code: ExperimentSignificanceCode + significant: bool + timezone: str + variants: list[ExperimentVariantFunnelsBaseStats] + + class CachedHogQLQueryResponse(BaseModel): model_config = ConfigDict( extra="forbid", @@ -4322,6 +4329,19 @@ class Response2(BaseModel): types: Optional[list] = Field(default=None, description="Types of returned columns") +class Response10(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + credible_intervals: dict[str, list[float]] + expected_loss: float + insight: FunnelsQueryResponse + probability: dict[str, float] + significance_code: ExperimentSignificanceCode + significant: bool + variants: list[ExperimentVariantFunnelsBaseStats] + + class DataWarehouseNode(BaseModel): model_config = ConfigDict( extra="forbid", @@ -4558,6 +4578,19 @@ class EventsNode(BaseModel): response: Optional[dict[str, Any]] = None +class ExperimentFunnelsQueryResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + credible_intervals: dict[str, list[float]] + expected_loss: float + insight: FunnelsQueryResponse + probability: dict[str, float] + significance_code: ExperimentSignificanceCode + significant: bool + variants: list[ExperimentVariantFunnelsBaseStats] + + class FunnelExclusionActionsNode(BaseModel): model_config = ConfigDict( extra="forbid", @@ -5468,18 +5501,18 @@ class EventsQuery(BaseModel): where: Optional[list[str]] = Field(default=None, description="HogQL filters to apply on returned data") -class ExperimentTrendQuery(BaseModel): +class ExperimentTrendsQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) count_query: TrendsQuery experiment_id: int exposure_query: Optional[TrendsQuery] = None - kind: Literal["ExperimentTrendQuery"] = "ExperimentTrendQuery" + kind: Literal["ExperimentTrendsQuery"] = "ExperimentTrendsQuery" modifiers: Optional[HogQLQueryModifiers] = Field( default=None, description="Modifiers used when performing the query" ) - response: Optional[ExperimentTrendQueryResponse] = None + response: Optional[ExperimentTrendsQueryResponse] = None class FunnelsQuery(BaseModel): @@ -5825,6 +5858,7 @@ class QueryResponseAlternative( QueryResponseAlternative25, QueryResponseAlternative26, QueryResponseAlternative27, + QueryResponseAlternative28, QueryResponseAlternative29, QueryResponseAlternative30, QueryResponseAlternative31, @@ -5864,6 +5898,7 @@ class QueryResponseAlternative( QueryResponseAlternative25, QueryResponseAlternative26, QueryResponseAlternative27, + QueryResponseAlternative28, QueryResponseAlternative29, QueryResponseAlternative30, QueryResponseAlternative31, @@ -5891,16 +5926,16 @@ class DatabaseSchemaQueryResponse(BaseModel): ] -class ExperimentFunnelQuery(BaseModel): +class ExperimentFunnelsQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) experiment_id: int - kind: Literal["ExperimentFunnelQuery"] = "ExperimentFunnelQuery" + kind: Literal["ExperimentFunnelsQuery"] = "ExperimentFunnelsQuery" modifiers: Optional[HogQLQueryModifiers] = Field( default=None, description="Modifiers used when performing the query" ) - response: Optional[ExperimentFunnelQueryResponse] = None + response: Optional[ExperimentFunnelsQueryResponse] = None source: FunnelsQuery @@ -6220,8 +6255,8 @@ class DataTableNode(BaseModel): WebGoalsQuery, SessionAttributionExplorerQuery, ErrorTrackingQuery, - ExperimentFunnelQuery, - ExperimentTrendQuery, + ExperimentFunnelsQuery, + ExperimentTrendsQuery, ] = Field(..., description="Source of the events") @@ -6260,8 +6295,8 @@ class HogQLAutocomplete(BaseModel): WebGoalsQuery, SessionAttributionExplorerQuery, ErrorTrackingQuery, - ExperimentFunnelQuery, - ExperimentTrendQuery, + ExperimentFunnelsQuery, + ExperimentTrendsQuery, ] ] = Field(default=None, description="Query in whose context to validate.") startPosition: int = Field(..., description="Start position of the editor word") @@ -6304,8 +6339,8 @@ class HogQLMetadata(BaseModel): WebGoalsQuery, SessionAttributionExplorerQuery, ErrorTrackingQuery, - ExperimentFunnelQuery, - ExperimentTrendQuery, + ExperimentFunnelsQuery, + ExperimentTrendsQuery, ] ] = Field( default=None, @@ -6346,8 +6381,8 @@ class QueryRequest(BaseModel): WebGoalsQuery, SessionAttributionExplorerQuery, ErrorTrackingQuery, - ExperimentFunnelQuery, - ExperimentTrendQuery, + ExperimentFunnelsQuery, + ExperimentTrendsQuery, DataVisualizationNode, DataTableNode, SavedInsightNode, @@ -6411,8 +6446,8 @@ class QuerySchemaRoot( WebGoalsQuery, SessionAttributionExplorerQuery, ErrorTrackingQuery, - ExperimentFunnelQuery, - ExperimentTrendQuery, + ExperimentFunnelsQuery, + ExperimentTrendsQuery, DataVisualizationNode, DataTableNode, SavedInsightNode, @@ -6450,8 +6485,8 @@ class QuerySchemaRoot( WebGoalsQuery, SessionAttributionExplorerQuery, ErrorTrackingQuery, - ExperimentFunnelQuery, - ExperimentTrendQuery, + ExperimentFunnelsQuery, + ExperimentTrendsQuery, DataVisualizationNode, DataTableNode, SavedInsightNode,