feat(hog): dict globals in autocomplete

ee/clickhouse/views/test/__snapshots__/test_clickhouse_experiments.ambr

@@ -13,26 +13,97 @@
 # ---
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results.1
   '''
-  /* celery:posthog.tasks.tasks.sync_insight_caching_state */
-  SELECT team_id,
-         date_diff('second', max(timestamp), now()) AS age
-  FROM events
-  WHERE timestamp > date_sub(DAY, 3, now())
-    AND timestamp < now()
-  GROUP BY team_id
-  ORDER BY age;
+  /* user_id:0 request:_snapshot_ */
+  SELECT array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS value,
+         count(*) as count
+  FROM events e
+  WHERE team_id = 2
+    AND event IN ['$pageleave', '$pageview']
+    AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC')
+    AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC')
+  GROUP BY value
+  ORDER BY count DESC, value DESC
+  LIMIT 26
+  OFFSET 0
   '''
 # ---
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results.2
   '''
-  /* celery:posthog.tasks.tasks.sync_insight_caching_state */
-  SELECT team_id,
-         date_diff('second', max(timestamp), now()) AS age
-  FROM events
-  WHERE timestamp > date_sub(DAY, 3, now())
-    AND timestamp < now()
-  GROUP BY team_id
-  ORDER BY age;
+  /* user_id:0 request:_snapshot_ */
+  SELECT countIf(steps = 1) step_1,
+         countIf(steps = 2) step_2,
+         avg(step_1_average_conversion_time_inner) step_1_average_conversion_time,
+         median(step_1_median_conversion_time_inner) step_1_median_conversion_time,
+         prop
+  FROM
+    (SELECT aggregation_target,
+            steps,
+            avg(step_1_conversion_time) step_1_average_conversion_time_inner,
+            median(step_1_conversion_time) step_1_median_conversion_time_inner ,
+            prop
+     FROM
+       (SELECT aggregation_target,
+               steps,
+               max(steps) over (PARTITION BY aggregation_target,
+                                             prop) as max_steps,
+                               step_1_conversion_time ,
+                               prop
+        FROM
+          (SELECT *,
+                  if(latest_0 <= latest_1
+                     AND latest_1 <= latest_0 + INTERVAL 14 DAY, 2, 1) AS steps ,
+                  if(isNotNull(latest_1)
+                     AND latest_1 <= latest_0 + INTERVAL 14 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time,
+                  prop
+           FROM
+             (SELECT aggregation_target, timestamp, step_0,
+                                                    latest_0,
+                                                    step_1,
+                                                    min(latest_1) over (PARTITION by aggregation_target,
+                                                                                     prop
+                                                                        ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_1 ,
+                                                                       if(has([['test'], ['control'], ['']], prop), prop, ['Other']) as prop
+              FROM
+                (SELECT *,
+                        if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['']) as prop
+                 FROM
+                   (SELECT e.timestamp as timestamp,
+                           pdi.person_id as aggregation_target,
+                           pdi.person_id as person_id,
+                           if(event = '$pageview', 1, 0) as step_0,
+                           if(step_0 = 1, timestamp, null) as latest_0,
+                           if(event = '$pageleave', 1, 0) as step_1,
+                           if(step_1 = 1, timestamp, null) as latest_1,
+                           array(replaceRegexpAll(JSONExtractRaw(properties, '$feature/a-b-test'), '^"|"$', '')) AS prop_basic,
+                           prop_basic as prop,
+                           argMinIf(prop, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop))) over (PARTITION by aggregation_target) as prop_vals
+                    FROM events e
+                    INNER JOIN
+                      (SELECT distinct_id,
+                              argMax(person_id, version) as person_id
+                       FROM person_distinct_id2
+                       WHERE team_id = 2
+                         AND distinct_id IN
+                           (SELECT distinct_id
+                            FROM events
+                            WHERE team_id = 2
+                              AND event IN ['$pageleave', '$pageview']
+                              AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC')
+                              AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC') )
+                       GROUP BY distinct_id
+                       HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id
+                    WHERE team_id = 2
+                      AND event IN ['$pageleave', '$pageview']
+                      AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC')
+                      AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-06 00:00:00', 'UTC')
+                      AND (step_0 = 1
+                           OR step_1 = 1) )))
+           WHERE step_0 = 1 ))
+     GROUP BY aggregation_target,
+              steps,
+              prop
+     HAVING steps = max_steps)
+  GROUP BY prop
   '''
 # ---
 # name: ClickhouseTestFunnelExperimentResults.test_experiment_flow_with_event_results.3

frontend/src/lib/components/HogQLEditor/HogQLEditor.tsx

@@ -44,7 +44,7 @@ export function HogQLEditor({
                 className={CLICK_OUTSIDE_BLOCK_CLASS}
                 minHeight="78px"
                 autoFocus={!disableAutoFocus}
-                metadataSource={metadataSource}
+                sourceQuery={metadataSource}
                 onPressCmdEnter={
                     disableCmdEnter
                         ? undefined

frontend/src/lib/monaco/CodeEditor.tsx

@@ -16,14 +16,15 @@ import { editor, editor as importedEditor, IDisposable } from 'monaco-editor'
 import { useEffect, useMemo, useRef, useState } from 'react'
 
 import { themeLogic } from '~/layout/navigation-3000/themeLogic'
-import { DataNode } from '~/queries/schema'
+import { AnyDataNode, HogLanguage } from '~/queries/schema'
 
 export interface CodeEditorProps extends Omit<EditorProps, 'loading' | 'theme'> {
     queryKey?: string
     autocompleteContext?: string
     onPressCmdEnter?: (value: string) => void
     autoFocus?: boolean
-    metadataSource?: DataNode
+    sourceQuery?: AnyDataNode
+    globals?: Record<string, any>
 }
 let codeEditorIndex = 0
 
@@ -47,7 +48,7 @@ function initEditor(
         }
     }
     if (editorProps?.language === 'hogQL' || editorProps?.language === 'hogQLExpr') {
-        const language: 'hogQL' | 'hogQLExpr' = editorProps.language
+        const language: HogLanguage = editorProps.language as HogLanguage
         if (!monaco.languages.getLanguages().some(({ id }) => id === language)) {
             monaco.languages.register(
                 language === 'hogQL'
@@ -75,7 +76,10 @@ function initEditor(
             })
             monaco.languages.setLanguageConfiguration('hogTemplate', hogTemplate.conf())
             monaco.languages.setMonarchTokensProvider('hogTemplate', hogTemplate.language())
-            monaco.languages.registerCompletionItemProvider('hogTemplate', hogQLAutocompleteProvider('hogTemplate'))
+            monaco.languages.registerCompletionItemProvider(
+                'hogTemplate',
+                hogQLAutocompleteProvider(HogLanguage.hogTemplate)
+            )
             monaco.languages.registerCodeActionProvider('hogTemplate', hogQLMetadataProvider())
         }
     }
@@ -116,7 +120,8 @@ export function CodeEditor({
     value,
     onPressCmdEnter,
     autoFocus,
-    metadataSource,
+    globals,
+    sourceQuery,
     ...editorProps
 }: CodeEditorProps): JSX.Element {
     const { isDarkModeOn } = useValues(themeLogic)
@@ -130,8 +135,9 @@ export function CodeEditor({
     const builtCodeEditorLogic = codeEditorLogic({
         key: queryKey ?? `new/${realKey}`,
         query: value ?? '',
-        language: editorProps.language,
-        metadataSource: metadataSource,
+        language: editorProps.language ?? 'text',
+        globals,
+        sourceQuery,
         monaco: monaco,
         editor: editor,
     })

frontend/src/lib/monaco/codeEditorLogic.tsx

@@ -11,11 +11,20 @@ import { loaders } from 'kea-loaders'
 import { editor, MarkerSeverity } from 'monaco-editor'
 
 import { performQuery } from '~/queries/query'
-import { DataNode, HogQLFilters, HogQLMetadata, HogQLMetadataResponse, HogQLNotice, NodeKind } from '~/queries/schema'
-import { isActorsQuery, isHogQLQuery } from '~/queries/utils'
+import {
+    AnyDataNode,
+    HogLanguage,
+    HogQLFilters,
+    HogQLMetadata,
+    HogQLMetadataResponse,
+    HogQLNotice,
+    NodeKind,
+} from '~/queries/schema'
 
 import type { codeEditorLogicType } from './codeEditorLogicType'
 
+const METADATA_LANGUAGES = [HogLanguage.hog, HogLanguage.hogQL, HogLanguage.hogQLExpr, HogLanguage.hogTemplate]
+
 export interface ModelMarker extends editor.IMarkerData {
     hogQLFix?: string
     start: number
@@ -25,19 +34,12 @@ export interface ModelMarker extends editor.IMarkerData {
 export interface CodeEditorLogicProps {
     key: string
     query: string
-    language?: string
-    metadataSource?: DataNode
+    language: string
+    sourceQuery?: AnyDataNode
     metadataFilters?: HogQLFilters
     monaco?: Monaco | null
     editor?: editor.IStandaloneCodeEditor | null
-}
-
-export function metadataSourceToQuery(metadataSource?: DataNode): string {
-    return metadataSource && isActorsQuery(metadataSource)
-        ? 'select * from persons'
-        : isHogQLQuery(metadataSource)
-        ? metadataSource.query
-        : 'select * from events'
+    globals?: Record<string, any>
 }
 
 export const codeEditorLogic = kea<codeEditorLogicType>([
@@ -53,37 +55,22 @@ export const codeEditorLogic = kea<codeEditorLogicType>([
             {
                 reloadMetadata: async (_, breakpoint) => {
                     const model = props.editor?.getModel()
-                    if (
-                        !model ||
-                        !props.monaco ||
-                        !['hog', 'hogQL', 'hogQLExpr', 'hogTemplate'].includes(props.language ?? '')
-                    ) {
+                    if (!model || !props.monaco || !METADATA_LANGUAGES.includes(props.language as HogLanguage)) {
                         return null
                     }
                     await breakpoint(300)
                     const query = props.query
                     if (query === '') {
                         return null
                     }
-                    const response = await performQuery<HogQLMetadata>(
-                        props.language === 'hogQL'
-                            ? { kind: NodeKind.HogQLMetadata, select: query, filters: props.metadataFilters }
-                            : props.language === 'hogTemplate'
-                            ? {
-                                  kind: NodeKind.HogQLMetadata,
-                                  template: query,
-                                  exprSource: metadataSourceToQuery(props.metadataSource),
-                                  filters: props.metadataFilters,
-                              }
-                            : props.language === 'hogQLExpr'
-                            ? {
-                                  kind: NodeKind.HogQLMetadata,
-                                  expr: query,
-                                  exprSource: metadataSourceToQuery(props.metadataSource),
-                                  filters: props.metadataFilters,
-                              }
-                            : { kind: NodeKind.HogQLMetadata, program: query, filters: props.metadataFilters }
-                    )
+                    const response = await performQuery<HogQLMetadata>({
+                        kind: NodeKind.HogQLMetadata,
+                        language: props.language as HogLanguage,
+                        query: query,
+                        filters: props.metadataFilters,
+                        globals: props.globals,
+                        sourceQuery: props.sourceQuery,
+                    })
                     breakpoint()
                     return [query, response]
                 },

frontend/src/lib/monaco/hogQLAutocompleteProvider.ts

@@ -1,10 +1,9 @@
 import { BuiltLogic } from 'kea'
-import { metadataSourceToQuery } from 'lib/monaco/codeEditorLogic'
 import type { codeEditorLogicType } from 'lib/monaco/codeEditorLogicType'
 import { languages } from 'monaco-editor'
 
 import { performQuery } from '~/queries/query'
-import { AutocompleteCompletionItem, HogQLAutocomplete, NodeKind } from '~/queries/schema'
+import { AutocompleteCompletionItem, HogLanguage, HogQLAutocomplete, NodeKind } from '~/queries/schema'
 
 const convertCompletionItemKind = (kind: AutocompleteCompletionItem['kind']): languages.CompletionItemKind => {
     switch (kind) {
@@ -79,9 +78,7 @@ const kindToSortText = (kind: AutocompleteCompletionItem['kind'], label: string)
     return `3-${label}`
 }
 
-export const hogQLAutocompleteProvider = (
-    type: 'hogQL' | 'hogQLExpr' | 'hogTemplate'
-): languages.CompletionItemProvider => ({
+export const hogQLAutocompleteProvider = (type: HogLanguage): languages.CompletionItemProvider => ({
     triggerCharacters: [' ', ',', '.', '{'],
     provideCompletionItems: async (model, position) => {
         const logic: BuiltLogic<codeEditorLogicType> | undefined = (model as any).codeEditorLogic
@@ -100,17 +97,14 @@ export const hogQLAutocompleteProvider = (
             lineNumber: position.lineNumber,
             column: word.endColumn,
         })
-        const metadataFilters = logic.isMounted() ? logic.props.metadataFilters : undefined
-        const exprSource = metadataSourceToQuery(logic.isMounted() ? logic.props.metadataSource : undefined)
         const query: HogQLAutocomplete = {
             kind: NodeKind.HogQLAutocomplete,
+            language: type,
             // Use the text from the model instead of logic due to a race condition on the logic values updating quick enough
-            ...(type === 'hogQL'
-                ? { select: model.getValue() }
-                : type === 'hogQLExpr'
-                ? { expr: model.getValue(), exprSource: exprSource }
-                : { template: model.getValue(), exprSource: exprSource }),
-            filters: metadataFilters,
+            query: model.getValue(),
+            filters: logic.isMounted() ? logic.props.metadataFilters : undefined,
+            globals: logic.isMounted() ? logic.props.globals : undefined,
+            sourceQuery: logic.isMounted() ? logic.props.sourceQuery : undefined,
             startPosition: startOffset,
             endPosition: endOffset,
         }